Diagnosed failure

CatalogManagerConfigurations/MasterStressTest.Test/0: WARNING: ThreadSanitizer: data race (pid=977)  Write of size 8 at 0x7b44000b4e48 by thread T127:
    #0 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::__destruct_at_end(kudu::MemTracker**) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:429:12 (libkudu_util.so+0x382770)
    #1 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libkudu_util.so+0x382684)
I20261120 03:00:52.121335  1180 consensus_queue.cc:261] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
    #2 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libkudu_util.so+0x38247b)
    #3 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libkudu_util.so+0x37a621)
    #4 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x37a1d3)
    #5 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x381495)
    #6 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libkudu_cli_actions.so+0x24d225)
    #7 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libkudu_cli_actions.so+0x24d1c9)
    #8 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #9 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x450864)
    #10 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x403c45)
    #11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x403e59)
    #12 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libkudu_cli_actions.so+0x2fbea7)
    #13 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libkudu_cli_actions.so+0x2fbe69)
    #14 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libkudu_cli_actions.so+0x2fbe29)
    #15 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libkudu_cli_actions.so+0x2fa1fa)
    #16 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3e1f09)
    #17 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3e1ee0)
    #18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3e1ea0)
    #19 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b4aa6)
    #20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b49c4)
    #21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b477b)
    #22 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x28c761)
    #23 kudu::tserver::TSTabletManager::PopulateIncrementalTabletReport(kudu::master::TabletReportPB*, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, bool) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1893:1 (libtserver.so+0x2c1b50)
    #24 kudu::tserver::Heartbeater::Thread::GenerateIncrementalTabletReport(kudu::master::TabletReportPB*, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:790:30 (libtserver.so+0x1cb58a)
    #25 kudu::tserver::Heartbeater::Thread::DoHeartbeat(kudu::master::MasterErrorPB*, kudu::rpc::ErrorStatusPB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:521:5 (libtserver.so+0x1cd4c4)
    #26 kudu::tserver::Heartbeater::Thread::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:645:21 (libtserver.so+0x1ce790)
    #27 kudu::tserver::Heartbeater::Thread::Start()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:728:48 (libtserver.so+0x1d03c1)
    #28 decltype(std::__1::forward<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(fp)()) std::__1::__invoke<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x1d0379)
    #29 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x1d0309)
    #30 std::__1::__function::__alloc_func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x1d02d1)
    #31 std::__1::__function::__func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x1cf5cd)
    #32 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #33 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #34 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Previous read of size 8 at 0x7b44000b4e48 by thread T144 (mutexes: write M299906313465758112):
    #0 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::end() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1536:30 (libkudu_util.so+0x37b53a)
    #1 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:284:22 (libkudu_util.so+0x378c16)
    #2 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x451230)
    #3 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:608:16 (libtablet.so+0x44a1c2)
    #4 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:579:5 (libtablet.so+0x449953)
    #5 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:65 (libtablet.so+0x44db61)
    #6 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x44db19)
    #7 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x44daa9)
    #8 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x44da71)
    #9 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x44cc9d)
    #10 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #11 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #12 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x4403a5)
    #13 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x442711)
    #14 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x4426c9)
    #15 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x442659)
    #16 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x442621)
I20261120 03:00:52.123251  1180 ts_tablet_manager.cc:1480] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.011s	user 0.002s	sys 0.003s
    #17 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x44191d)
    #18 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #19 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #20 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Location is heap block of size 264 at 0x7b44000b4dc0 allocated by thread T135:
    #0 operator new(unsigned long) /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:64 (kudu+0x320f47)
    #1 std::__1::__libcpp_allocate(unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:253:10 (libkudu_util.so+0x2ae876)
    #2 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::allocate(unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1789:34 (libkudu_util.so+0x380d75)
    #3 std::__1::enable_if<!(is_array<long&>::value), std::__1::shared_ptr<long&> >::type std::__1::make_shared<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler&&...) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4290:45 (libkudu_util.so+0x380b51)
    #4 std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/make_shared.h:61:12 (libkudu_util.so+0x37a485)
    #5 kudu::MemTracker::CreateTracker(long, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:54:34 (libkudu_util.so+0x3784ae)
    #6 kudu::tablet::OpTracker::StartMemoryTracking(std::__1::shared_ptr<kudu::MemTracker> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:311:20 (libtablet.so+0x451ceb)
    #7 kudu::tablet::TabletReplica::Start(kudu::consensus::ConsensusBootstrapInfo const&, std::__1::shared_ptr<kudu::tablet::Tablet>, kudu::clock::Clock*, std::__1::shared_ptr<kudu::rpc::Messenger>, scoped_refptr<kudu::rpc::ResultTracker>, scoped_refptr<kudu::log::Log>, kudu::consensus::MultiRaftManager*, kudu::ThreadPool*, kudu::DnsResolver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:278:19 (libtablet.so+0x404ca6)
    #8 kudu::tserver::TSTabletManager::OpenTablet(scoped_refptr<kudu::tablet::TabletReplica> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&, std::__1::atomic<int>*, std::__1::atomic<int>*, kudu::Timer*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1482:18 (libtserver.so+0x2bd7ba)
    #9 kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:755:3 (libtserver.so+0x2d2901)
    #10 decltype(std::__1::forward<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12&>(fp)()) std::__1::__invoke<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12&>(kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x2d28a9)
    #11 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12&>(kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x2d2839)
    #12 std::__1::__function::__alloc_func<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12, std::__1::allocator<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x2d2801)
    #13 std::__1::__function::__func<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12, std::__1::allocator<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x2d1add)
    #14 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #15 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #16 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x4403a5)
    #17 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x442711)
    #18 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x4426c9)
    #19 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x442659)
    #20 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x442621)
    #21 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x44191d)
    #22 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #23 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #24 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Mutex M299906313465758112 is already destroyed.

  Thread T127 'heartbeat-1124' (tid=1124, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::tserver::Heartbeater::Thread::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:727:10 (libtserver.so+0x1cae61)
    #4 kudu::tserver::Heartbeater::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:251:41 (libtserver.so+0x1cac3d)
    #5 kudu::tserver::TabletServer::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:167:3 (libtserver.so+0x259ccc)
    #6 kudu::tserver::RunTabletServer() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server_runner.cc:86:3 (libtserver.so+0x266e93)
    #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:128:10 (libkudu_cli_actions.so+0x3b6429)
    #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_cli_actions.so+0x28c051)
    #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (libkudu_cli_actions.so+0x28bfbc)
    #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_cli_actions.so+0x28bf2c)
    #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_cli_actions.so+0x28ab95)
    #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x14082f)
    #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132da1)
    #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:270:10 (libkudu_tools_util.so+0x130a52)
    #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:114:15 (kudu+0x324910)
    #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:186:16 (kudu+0x325247)
    #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:249:10 (kudu+0x325671)

  Thread T144 'apply [worker]-' (tid=1286, running) created by thread T82 at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x43f5cb)
    #4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x43d8cb)
    #5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x43fb32)
    #6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:23 (libtablet.so+0x448cf0)
    #7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:476:5 (libtablet.so+0x449289)
    #8 kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:860:15 (libtablet.so+0x41362a)
    #9 decltype(std::__1::forward<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x4135b8)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x413538)
    #11 std::__1::__function::__alloc_func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x4134f3)
    #12 std::__1::__function::__func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41270c)
    #13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x40fe06)
    #14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x40a5b8)
    #15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3598:3 (libconsensus.so+0x1fd9c0)
    #16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:200:12 (libconsensus.so+0x1d2e9d)
    #17 kudu::consensus::RaftConsensus::UpdateReplica(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::consensus::RaftConsensus::FlushContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1603:5 (libconsensus.so+0x1f0713)
    #18 kudu::consensus::RaftConsensus::Update(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1162:9 (libconsensus.so+0x1efd04)
    #19 kudu::tserver::ConsensusServiceImpl::UpdateConsensus(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1792:25 (libtserver.so+0x26e199)
    #20 kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/consensus/consensus.service.cc:321:13 (libconsensus_proto.so+0xd8d34)
    #21 decltype(std::__1::forward<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus_proto.so+0xd8cc2)
    #22 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus_proto.so+0xd8bf1)
    #23 std::__1::__function::__alloc_func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus_proto.so+0xd8b6c)
    #24 std::__1::__function::__func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus_proto.so+0xd7e22)
    #25 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x21b0bc)
    #26 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x21a4f6)
    #27 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x219e9f)
    #28 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15 (libkrpc.so+0x21d16f)
    #29 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5 (libkrpc.so+0x21e4c1)
    #30 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x21e479)
    #31 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x21e409)
    #32 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x21e3d1)
    #33 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x21d6cd)
    #34 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #35 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #36 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Thread T135 'tablet-open [wo' (tid=1181, running) created by thread T62 at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x43f5cb)
    #4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x43d8cb)
    #5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x43fb32)
    #6 kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:755:3 (libtserver.so+0x2ba982)
    #7 kudu::tserver::TabletServiceAdminImpl::CreateTablet(kudu::tserver::CreateTabletRequestPB const*, kudu::tserver::CreateTabletResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1502:34 (libtserver.so+0x26a180)
    #8 kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/tserver/tserver_admin.service.cc:189:13 (libtserver_admin_proto.so+0x68bf4)
    #9 decltype(std::__1::forward<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver_admin_proto.so+0x68b82)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver_admin_proto.so+0x68ab1)
    #11 std::__1::__function::__alloc_func<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver_admin_proto.so+0x68a2c)
    #12 std::__1::__function::__func<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver_admin_proto.so+0x67ce2)
    #13 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x21b0bc)
    #14 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x21a4f6)
    #15 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x219e9f)
    #16 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15 (libkrpc.so+0x21d16f)
    #17 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5 (libkrpc.so+0x21e4c1)
    #18 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x21e479)
    #19 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x21e409)
    #20 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x21e3d1)
    #21 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x21d6cd)
    #22 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #23 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #24 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)


CatalogManagerConfigurations/MasterStressTest.Test/0: WARNING: ThreadSanitizer: data race (pid=977)  Write of size 8 at 0x7b08000d5460 by thread T127:
    #0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (kudu+0x321819)
    #1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (kudu+0x3287c9)
    #2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (kudu+0x328769)
    #3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x2aebd2)
    #4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x2aeb69)
    #5 std::__1::allocator<kudu::MemTracker*>::deallocate(kudu::MemTracker**, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x38284a)
    #6 std::__1::allocator_traits<std::__1::allocator<kudu::MemTracker*> >::deallocate(std::__1::allocator<kudu::MemTracker*>&, kudu::MemTracker**, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1533:14 (libkudu_util.so+0x3826c9)
    #7 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:466:9 (libkudu_util.so+0x3824a7)
    #8 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libkudu_util.so+0x37a621)
    #9 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x37a1d3)
    #10 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x381495)
    #11 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libkudu_cli_actions.so+0x24d225)
    #12 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libkudu_cli_actions.so+0x24d1c9)
    #13 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #14 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x450864)
    #15 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x403c45)
    #16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x403e59)
    #17 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libkudu_cli_actions.so+0x2fbea7)
    #18 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libkudu_cli_actions.so+0x2fbe69)
    #19 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libkudu_cli_actions.so+0x2fbe29)
    #20 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libkudu_cli_actions.so+0x2fa1fa)
    #21 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3e1f09)
    #22 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3e1ee0)
    #23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3e1ea0)
    #24 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b4aa6)
    #25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b49c4)
    #26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b477b)
    #27 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x28c761)
    #28 kudu::tserver::TSTabletManager::PopulateIncrementalTabletReport(kudu::master::TabletReportPB*, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, bool) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1893:1 (libtserver.so+0x2c1b50)
    #29 kudu::tserver::Heartbeater::Thread::GenerateIncrementalTabletReport(kudu::master::TabletReportPB*, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:790:30 (libtserver.so+0x1cb58a)
    #30 kudu::tserver::Heartbeater::Thread::DoHeartbeat(kudu::master::MasterErrorPB*, kudu::rpc::ErrorStatusPB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:521:5 (libtserver.so+0x1cd4c4)
    #31 kudu::tserver::Heartbeater::Thread::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:645:21 (libtserver.so+0x1ce790)
    #32 kudu::tserver::Heartbeater::Thread::Start()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:728:48 (libtserver.so+0x1d03c1)
    #33 decltype(std::__1::forward<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(fp)()) std::__1::__invoke<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x1d0379)
    #34 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x1d0309)
    #35 std::__1::__function::__alloc_func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x1d02d1)
    #36 std::__1::__function::__func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x1cf5cd)
    #37 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #38 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #39 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Previous read of size 8 at 0x7b08000d5460 by thread T144 (mutexes: write M299906313465758112):
    #0 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:5 (libkudu_util.so+0x378c52)
    #1 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x451230)
    #2 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:608:16 (libtablet.so+0x44a1c2)
    #3 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:579:5 (libtablet.so+0x449953)
    #4 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:65 (libtablet.so+0x44db61)
    #5 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x44db19)
    #6 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x44daa9)
    #7 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x44da71)
    #8 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x44cc9d)
I20261120 03:00:52.177731  1280 raft_consensus.cc:534] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
    #9 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #10 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #11 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x4403a5)
    #12 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x442711)
    #13 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x4426c9)
    #14 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x442659)
    #15 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x442621)
    #16 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x44191d)
    #17 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #18 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #19 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Mutex M299906313465758112 is already destroyed.

  Thread T127 'heartbeat-1124' (tid=1124, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::tserver::Heartbeater::Thread::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:727:10 (libtserver.so+0x1cae61)
    #4 kudu::tserver::Heartbeater::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:251:41 (libtserver.so+0x1cac3d)
    #5 kudu::tserver::TabletServer::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:167:3 (libtserver.so+0x259ccc)
    #6 kudu::tserver::RunTabletServer() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server_runner.cc:86:3 (libtserver.so+0x266e93)
    #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:128:10 (libkudu_cli_actions.so+0x3b6429)
    #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_cli_actions.so+0x28c051)
    #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (libkudu_cli_actions.so+0x28bfbc)
    #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_cli_actions.so+0x28bf2c)
    #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_cli_actions.so+0x28ab95)
    #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x14082f)
    #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132da1)
    #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:270:10 (libkudu_tools_util.so+0x130a52)
    #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:114:15 (kudu+0x324910)
    #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:186:16 (kudu+0x325247)
    #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:249:10 (kudu+0x325671)

  Thread T144I20261120 03:00:52.178210  1280 raft_consensus.cc:556] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
 'apply [worker]-' (tid=1286, running) created by thread T82 at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x43f5cb)
    #4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x43d8cb)
    #5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x43fb32)
    #6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:23 (libtablet.so+0x448cf0)
    #7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:476:5 (libtablet.so+0x449289)
    #8 kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:860:15 (libtablet.so+0x41362a)
    #9 decltype(std::__1::forward<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x4135b8)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x413538)
    #11 std::__1::__function::__alloc_func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x4134f3)
    #12 std::__1::__function::__func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41270c)
    #13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x40fe06)
    #14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x40a5b8)
    #15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3598:3 (libconsensus.so+0x1fd9c0)
    #16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:200:12 (libconsensus.so+0x1d2e9d)
    #17 kudu::consensus::RaftConsensus::UpdateReplica(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::consensus::RaftConsensus::FlushContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1603:5 (libconsensus.so+0x1f0713)
    #18 kudu::consensus::RaftConsensus::Update(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1162:9 (libconsensus.so+0x1efd04)
    #19 kudu::tserver::ConsensusServiceImpl::UpdateConsensus(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1792:25 (libtserver.so+0x26e199)
    #20 kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/consensus/consensus.service.cc:321:13 (libconsensus_proto.so+0xd8d34)
    #21 decltype(std::__1::forward<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus_proto.so+0xd8cc2)
    #22 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus_proto.so+0xd8bf1)
    #23 std::__1::__function::__alloc_func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus_proto.so+0xd8b6c)
    #24 std::__1::__function::__func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus_proto.so+0xd7e22)
    #25 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x21b0bc)
    #26 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x21a4f6)
    #27 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x219e9f)
    #28 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15 (libkrpc.so+0x21d16f)
    #29 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5 (libkrpc.so+0x21e4c1)
    #30 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x21e479)
    #31 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x21e409)
    #32 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x21e3d1)
    #33 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x21d6cd)
    #34 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #35 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #36 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)


CatalogManagerConfigurations/MasterStressTest.Test/0: WARNING: ThreadSanitizer: data race (pid=977)  Write of size 8 at 0x7b44000b4570 by thread T127:
I20261120 03:00:52.611820  1044 tablet_service.cc:1542] Processing DeleteTablet for tablet fa57d0c282964e37b9133ea57f458a51 with delete_type TABLET_DATA_DELETED (replaced by tablet c6a65c1a4a574961bec657ad7c9048e0) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:52.612568  1045 tablet_service.cc:1495] Processing CreateTablet for tablet 68179488b9bc4e03b65a9f1d434c95b5 (DEFAULT_TABLE table=default.Table_650dc67e0ac74c149a3f64d456767938 [id=cfcd5fea97fb4ab3baff99eb28cd9476]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.618956  1289 consensus_queue.cc:1049] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:52.622421   500 master_service.cc:965] ReplaceTablet: received request to replace tablet fa57d0c282964e37b9133ea57f458a51 from {username='slave'} at 127.0.0.1:45640
I20261120 03:00:52.627468   857 leader_election.cc:304] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:52.627823  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00dd2ffad1124244b171b2657d38a708" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:52.628221  1380 raft_consensus.cc:3024] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:52.628666  1380 raft_consensus.cc:744] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:00:52.629410  1380 consensus_queue.cc:238] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:52.631420  1180 tablet_bootstrap.cc:492] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:52.631793  1180 ts_tablet_manager.cc:1449] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.283s	user 0.010s	sys 0.001s
    #0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (kudu+0x321819)
    #1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (kudu+0x3287c9)
    #2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (kudu+0x328769)
    #3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x2aebd2)
    #4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x2aeb69)
    #5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x381bcd)
    #6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x381500)
    #7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
    #8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libkudu_cli_actions.so+0x24d1d5)
    #9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #10 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x37a1db)
    #11 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x381495)
    #12 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libkudu_cli_actions.so+0x24d225)
    #13 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libkudu_cli_actions.so+0x24d1c9)
    #14 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #15 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x450864)
    #16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x403c45)
    #17 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x403e59)
    #18 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libkudu_cli_actions.so+0x2fbea7)
    #19 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libkudu_cli_actions.so+0x2fbe69)
    #20 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libkudu_cli_actions.so+0x2fbe29)
    #21 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libkudu_cli_actions.so+0x2fa1fa)
    #22 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3e1f09)
    #23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3e1ee0)
    #24 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3e1ea0)
    #25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b4aa6)
    #26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b49c4)
    #27 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b477b)
    #28 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x28c761)
    #29 kudu::tserver::TSTabletManager::PopulateIncrementalTabletReport(kudu::master::TabletReportPB*, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, bool) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1893:1 (libtserver.so+0x2c1b50)
    #30 kudu::tserver::Heartbeater::Thread::GenerateIncrementalTabletReport(kudu::master::TabletReportPB*, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:790:30 (libtserver.so+0x1cb58a)
    #31 kudu::tserver::Heartbeater::Thread::DoHeartbeat(kudu::master::MasterErrorPB*, kudu::rpc::ErrorStatusPB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:521:5 (libtserver.so+0x1cd4c4)
    #32 kudu::tserver::Heartbeater::Thread::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:645:21 (libtserver.so+0x1ce790)
    #33 kudu::tserver::Heartbeater::Thread::Start()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:728:48 (libtserver.so+0x1d03c1)
    #34 decltype(std::__1::forward<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(fp)()) std::__1::__invoke<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x1d0379)
    #35 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x1d0309)
    #36 std::__1::__function::__alloc_func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x1d02d1)
    #37 std::__1::__function::__func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x1cf5cd)
    #38 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #39 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #40 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Previous atomic write of size 8 at 0x7b44000b4570 by thread T144 (mutexes: write M299906313465758112):
    #0 __tsan_atomic64_fetch_add /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:620 (kudu+0x2ca8f5)
    #1 long std::__1::__cxx_atomic_fetch_add<long>(std::__1::__cxx_atomic_base_impl<long>*, long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1042:12 (libkudu_util.so+0x2c2cd3)
    #2 std::__1::__atomic_base<long, true>::fetch_add(long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1706:17 (libkudu_util.so+0x2c2c09)
    #3 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:30 (libkudu_util.so+0x37b5f2)
    #4 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x378c61)
    #5 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x451230)
    #6 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:608:16 (libtablet.so+0x44a1c2)
    #7 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:579:5 (libtablet.so+0x449953)
    #8 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:65 (libtablet.so+0x44db61)
    #9 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x44db19)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x44daa9)
    #11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x44da71)
    #12 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x44cc9d)
    #13 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #14 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #15 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x4403a5)
    #16 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x442711)
I20261120 03:00:52.638913  1324 tablet_bootstrap.cc:492] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:52.639336  1324 ts_tablet_manager.cc:1449] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.407s	user 0.011s	sys 0.007s
    #17 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x4426c9)
    #18 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x442659)
    #19 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x442621)
    #20 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x44191d)
    #21 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #22 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #23 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Mutex M299906313465758112 is already destroyed.

  Thread T127 'heartbeat-1124' (tid=1124, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::tserver::Heartbeater::Thread::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:727:10 (libtserver.so+0x1cae61)
I20261120 03:00:52.649816  1400 raft_consensus.cc:534] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
    #4 kudu::tserver::Heartbeater::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:251:41 (libtserver.so+0x1cac3d)
    #5 kudu::tserver::TabletServer::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:167:3 (libtserver.so+0x259ccc)
    #6 kudu::tserver::RunTabletServer() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server_runner.cc:86:3 (libtserver.so+0x266e93)
I20261120 03:00:52.650552  1400 raft_consensus.cc:556] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
    #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:128:10 (libkudu_cli_actions.so+0x3b6429)
    #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_cli_actions.so+0x28c051)
    #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (libkudu_cli_actions.so+0x28bfbc)
    #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_cli_actions.so+0x28bf2c)
    #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_cli_actions.so+0x28ab95)
    #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x14082f)
    #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132da1)
    #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:270:10 (libkudu_tools_util.so+0x130a52)
    #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:114:15 (kudu+0x324910)
    #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:186:16 (kudu+0x325247)
    #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:249:10 (kudu+0x325671)

  Thread T144 'apply [worker]-' (tid=1286, finished) created by thread T82 at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x43f5cb)
    #4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x43d8cb)
    #5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x43fb32)
    #6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:23 (libtablet.so+0x448cf0)
    #7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:476:5 (libtablet.so+0x449289)
    #8 kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:860:15 (libtablet.so+0x41362a)
    #9 decltype(std::__1::forward<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x4135b8)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x413538)
    #11 std::__1::__function::__alloc_func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x4134f3)
    #12 std::__1::__function::__func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41270c)
W20261120 03:00:52.661569   709 leader_election.cc:343] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
    #13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x40fe06)
    #14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x40a5b8)
    #15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3598:3 (libconsensus.so+0x1fd9c0)
    #16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:200:12 (libconsensus.so+0x1d2e9d)
    #17 kudu::consensus::RaftConsensus::UpdateReplica(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::consensus::RaftConsensus::FlushContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1603:5 (libconsensus.so+0x1f0713)
    #18 kudu::consensus::RaftConsensus::Update(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1162:9 (libconsensus.so+0x1efd04)
I20261120 03:00:52.664889  1400 leader_election.cc:290] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:52.665447   493 catalog_manager.cc:6146] Not sending DeleteTablet requests; no consensus state for tablet 5144400be7ee47e783e1266eac8ded62
    #19 kudu::tserver::ConsensusServiceImpl::UpdateConsensus(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1792:25 (libtserver.so+0x26e199)
    #20 kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/consensus/consensus.service.cc:321:13 (libconsensus_proto.so+0xd8d34)
    #21 decltype(std::__1::forward<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus_proto.so+0xd8cc2)
    #22 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus_proto.so+0xd8bf1)
    #23 std::__1::__function::__alloc_func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus_proto.so+0xd8b6c)
I20261120 03:00:52.669150  1180 raft_consensus.cc:366] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
    #24 std::__1::__function::__func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus_proto.so+0xd7e22)
    #25 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x21b0bc)
    #26 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x21a4f6)
I20261120 03:00:52.641776  1324 raft_consensus.cc:366] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:52.673517  1324 raft_consensus.cc:392] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:52.673781  1324 raft_consensus.cc:787] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
    #27 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x219e9f)
    #28 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15 (libkrpc.so+0x21d16f)
    #29 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5 (libkrpc.so+0x21e4c1)
    #30 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x21e479)
    #31 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x21e409)
    #32 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x21e3d1)
    #33 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x21d6cd)
    #34 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #35 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #36 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)


CatalogManagerConfigurations/MasterStressTest.Test/0: WARNING: ThreadSanitizer: data race (pid=977)  Write of size 8 at 0x7b44000b4578 by thread T127:
    #0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (kudu+0x321819)
    #1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (kudu+0x3287c9)
    #2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (kudu+0x328769)
    #3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x2aebd2)
    #4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x2aeb69)
    #5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x381bcd)
    #6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x381500)
    #7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
    #8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libkudu_cli_actions.so+0x24d1d5)
    #9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #10 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x37a1db)
    #11 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x381495)
    #12 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libkudu_cli_actions.so+0x24d225)
    #13 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libkudu_cli_actions.so+0x24d1c9)
    #14 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #15 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x450864)
    #16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x403c45)
    #17 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x403e59)
    #18 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libkudu_cli_actions.so+0x2fbea7)
    #19 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libkudu_cli_actions.so+0x2fbe69)
    #20 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libkudu_cli_actions.so+0x2fbe29)
    #21 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libkudu_cli_actions.so+0x2fa1fa)
    #22 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3e1f09)
    #23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3e1ee0)
    #24 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3e1ea0)
    #25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b4aa6)
    #26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b49c4)
    #27 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b477b)
    #28 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x28c761)
    #29 kudu::tserver::TSTabletManager::PopulateIncrementalTabletReport(kudu::master::TabletReportPB*, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, bool) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1893:1 (libtserver.so+0x2c1b50)
    #30 kudu::tserver::Heartbeater::Thread::GenerateIncrementalTabletReport(kudu::master::TabletReportPB*, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:790:30 (libtserver.so+0x1cb58a)
    #31 kudu::tserver::Heartbeater::Thread::DoHeartbeat(kudu::master::MasterErrorPB*, kudu::rpc::ErrorStatusPB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:521:5 (libtserver.so+0x1cd4c4)
    #32 kudu::tserver::Heartbeater::Thread::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:645:21 (libtserver.so+0x1ce790)
    #33 kudu::tserver::Heartbeater::Thread::Start()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:728:48 (libtserver.so+0x1d03c1)
    #34 decltype(std::__1::forward<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(fp)()) std::__1::__invoke<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x1d0379)
    #35 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x1d0309)
    #36 std::__1::__function::__alloc_func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x1d02d1)
    #37 std::__1::__function::__func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x1cf5cd)
    #38 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #39 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #40 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Previous atomic write of size 8 at 0x7b44000b4578 by thread T144 (mutexes: write M299906313465758112):
    #0 __tsan_atomic64_compare_exchange_val /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:850 (kudu+0x2d19c8)
    #1 bool std::__1::__cxx_atomic_compare_exchange_weak<long>(std::__1::__cxx_atomic_base_impl<long>*, long*, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1031:12 (liblog.so+0x8c014)
    #2 std::__1::__atomic_base<long, false>::compare_exchange_weak(long&, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1622:17 (liblog.so+0x8be39)
    #3 void kudu::AtomicStoreMax<long>(std::__1::atomic<long>&, long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/atomic-utils.h:33:13 (liblog.so+0x7ec80)
    #4 kudu::HighWaterMark::UpdateMax(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:80:5 (libkudu_util.so+0x380794)
    #5 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:5 (libkudu_util.so+0x37b5fd)
    #6 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x378c61)
    #7 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x451230)
    #8 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:608:16 (libtablet.so+0x44a1c2)
    #9 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:579:5 (libtablet.so+0x449953)
    #10 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:65 (libtablet.so+0x44db61)
    #11 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x44db19)
    #12 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x44daa9)
    #13 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x44da71)
    #14 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x44cc9d)
    #15 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #16 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #17 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x4403a5)
    #18 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x442711)
    #19 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x4426c9)
    #20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x442659)
    #21 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x442621)
    #22 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x44191d)
    #23 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #24 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #25 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

I20261120 03:00:52.955022  1180 tablet_bootstrap.cc:700] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440: Time spent opening tablet: real 0.153s	user 0.004s	sys 0.000s
I20261120 03:00:52.890875  1400 raft_consensus.cc:556] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:52.959445  1400 leader_election.cc:290] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:52.904310  1185 tablet_replica.cc:333] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:52.934953  1289 leader_election.cc:290] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:52.960928  1185 raft_consensus.cc:2440] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:52.961850  1185 raft_consensus.cc:2469] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
  Mutex M299906313465758112 is already destroyed.

  Thread T127 'heartbeat-1124' (tid=1124, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::tserver::Heartbeater::Thread::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:727:10 (libtserver.so+0x1cae61)
    #4 kudu::tserver::Heartbeater::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:251:41 (libtserver.so+0x1cac3d)
    #5 kudu::tserver::TabletServer::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:167:3 (libtserver.so+0x259ccc)
    #6 kudu::tserver::RunTabletServer() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server_runner.cc:86:3 (libtserver.so+0x266e93)
    #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:128:10 (libkudu_cli_actions.so+0x3b6429)
    #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_cli_actions.so+0x28c051)
    #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (libkudu_cli_actions.so+0x28bfbc)
    #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_cli_actions.so+0x28bf2c)
    #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_cli_actions.so+0x28ab95)
    #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x14082f)
    #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132da1)
    #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:270:10 (libkudu_tools_util.so+0x130a52)
    #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:114:15 (kudu+0x324910)
    #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:186:16 (kudu+0x325247)
    #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:249:10 (kudu+0x325671)

  Thread T144 'apply [worker]-' (tid=1286, finished) created by thread T82 at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x43f5cb)
    #4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x43d8cb)
    #5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x43fb32)
    #6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:23 (libtablet.so+0x448cf0)
    #7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:476:5 (libtablet.so+0x449289)
    #8 kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:860:15 (libtablet.so+0x41362a)
    #9 decltype(std::__1::forward<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x4135b8)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x413538)
    #11 std::__1::__function::__alloc_func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x4134f3)
    #12 std::__1::__function::__func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41270c)
    #13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x40fe06)
    #14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x40a5b8)
    #15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3598:3 (libconsensus.so+0x1fd9c0)
    #16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:200:12 (libconsensus.so+0x1d2e9d)
    #17 kudu::consensus::RaftConsensus::UpdateReplica(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::consensus::RaftConsensus::FlushContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1603:5 (libconsensus.so+0x1f0713)
    #18 kudu::consensus::RaftConsensus::Update(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1162:9 (libconsensus.so+0x1efd04)
    #19 kudu::tserver::ConsensusServiceImpl::UpdateConsensus(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1792:25 (libtserver.so+0x26e199)
    #20 kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/consensus/consensus.service.cc:321:13 (libconsensus_proto.so+0xd8d34)
    #21 decltype(std::__1::forward<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus_proto.so+0xd8cc2)
    #22 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus_proto.so+0xd8bf1)
    #23 std::__1::__function::__alloc_func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus_proto.so+0xd8b6c)
    #24 std::__1::__function::__func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus_proto.so+0xd7e22)
    #25 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x21b0bc)
    #26 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x21a4f6)
    #27 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x219e9f)
    #28 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15 (libkrpc.so+0x21d16f)
    #29 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5 (libkrpc.so+0x21e4c1)
    #30 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x21e479)
    #31 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x21e409)
    #32 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x21e3d1)
    #33 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x21d6cd)
    #34 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #35 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #36 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)


CatalogManagerConfigurations/MasterStressTest.Test/0: WARNING: ThreadSanitizer: data race (pid=977)  Write of size 8 at 0x7b44000b4e30 by thread T127:
    #0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (kudu+0x321819)
I20261120 03:00:53.216331  1392 ts_tablet_manager.cc:1996] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:53.219919   501 master_service.cc:965] ReplaceTablet: received request to replace tablet 020be7f7cda0467aab56f657cbe8d0ad from {username='slave'} at 127.0.0.1:45640
    #1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (kudu+0x3287c9)
    #2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (kudu+0x328769)
    #3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x2aebd2)
    #4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x2aeb69)
    #5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x381bcd)
    #6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x381500)
    #7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
    #8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libkudu_cli_actions.so+0x24d1d5)
    #9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x450864)
    #11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x403c45)
    #12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x403e59)
    #13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libkudu_cli_actions.so+0x2fbea7)
    #14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libkudu_cli_actions.so+0x2fbe69)
    #15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libkudu_cli_actions.so+0x2fbe29)
I20261120 03:00:53.223577  1185 tablet_replica.cc:333] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
    #16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libkudu_cli_actions.so+0x2fa1fa)
    #17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3e1f09)
    #18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3e1ee0)
I20261120 03:00:53.224277  1185 raft_consensus.cc:2440] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Raft consensus shutting down.
    #19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3e1ea0)
I20261120 03:00:53.224895  1185 pending_rounds.cc:70] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: Trying to abort 1 pending ops.
    #20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b4aa6)
    #21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b49c4)
I20261120 03:00:53.224547  1392 tablet_replica.cc:333] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:53.227474   493 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_a97b43669b2d4944ad7c61d64e5a2cbd"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:53.229270   493 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_a97b43669b2d4944ad7c61d64e5a2cbd in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
    #22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b477b)
I20261120 03:00:53.230686  1185 pending_rounds.cc:77] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7352908402572042240 op_type: NO_OP noop_request { }
    #23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x28c761)
    #24 kudu::tserver::TSTabletManager::PopulateIncrementalTabletReport(kudu::master::TabletReportPB*, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, bool) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1893:1 (libtserver.so+0x2c1b50)
I20261120 03:00:53.234473  1185 raft_consensus.cc:3115] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
    #25 kudu::tserver::Heartbeater::Thread::GenerateIncrementalTabletReport(kudu::master::TabletReportPB*, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:790:30 (libtserver.so+0x1cb58a)
    #26 kudu::tserver::Heartbeater::Thread::DoHeartbeat(kudu::master::MasterErrorPB*, kudu::rpc::ErrorStatusPB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:521:5 (libtserver.so+0x1cd4c4)
I20261120 03:00:53.234943  1185 raft_consensus.cc:2469] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
    #27 kudu::tserver::Heartbeater::Thread::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:645:21 (libtserver.so+0x1ce790)
    #28 kudu::tserver::Heartbeater::Thread::Start()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:728:48 (libtserver.so+0x1d03c1)
    #29 decltype(std::__1::forward<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(fp)()) std::__1::__invoke<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x1d0379)
    #30 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x1d0309)
    #31 std::__1::__function::__alloc_func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x1d02d1)
    #32 std::__1::__function::__func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x1cf5cd)
    #33 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #34 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #35 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Previous atomic write of size 8 at 0x7b44000b4e30 by thread T144 (mutexes: write M299906313465758112):
    #0 __tsan_atomic64_fetch_add /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:620 (kudu+0x2ca8f5)
    #1 long std::__1::__cxx_atomic_fetch_add<long>(std::__1::__cxx_atomic_base_impl<long>*, long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1042:12 (libkudu_util.so+0x2c2cd3)
    #2 std::__1::__atomic_base<long, true>::fetch_add(long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1706:17 (libkudu_util.so+0x2c2c09)
I20261120 03:00:53.237917  1185 ts_tablet_manager.cc:1962] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
    #3 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:30 (libkudu_util.so+0x37b5f2)
    #4 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x378c61)
    #5 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x451230)
    #6 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:608:16 (libtablet.so+0x44a1c2)
    #7 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:579:5 (libtablet.so+0x449953)
    #8 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:65 (libtablet.so+0x44db61)
    #9 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x44db19)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x44daa9)
    #11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x44da71)
    #12 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x44cc9d)
    #13 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #14 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #15 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x4403a5)
    #16 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x442711)
    #17 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x4426c9)
    #18 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x442659)
    #19 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x442621)
    #20 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x44191d)
    #21 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #22 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #23 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Mutex M299906313465758112 is already destroyed.

  Thread T127 'heartbeat-1124' (tid=1124, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::tserver::Heartbeater::Thread::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:727:10 (libtserver.so+0x1cae61)
I20261120 03:00:53.244670   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet fa57d0c282964e37b9133ea57f458a51 (table default.Table_dd2f7617ff0e401b9b862d5991110dd5 [id=3955f41b6d4d41bf8ba2c81e4ac7f4a8]) successfully deleted
    #4 kudu::tserver::Heartbeater::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:251:41 (libtserver.so+0x1cac3d)
I20261120 03:00:53.244541  1392 raft_consensus.cc:2440] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
    #5 kudu::tserver::TabletServer::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:167:3 (libtserver.so+0x259ccc)
    #6 kudu::tserver::RunTabletServer() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server_runner.cc:86:3 (libtserver.so+0x266e93)
    #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:128:10 (libkudu_cli_actions.so+0x3b6429)
    #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_cli_actions.so+0x28c051)
I20261120 03:00:53.245923  1392 raft_consensus.cc:2469] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:53.246049   499 catalog_manager.cc:5856] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65). New cstate: current_term: 1 leader_uuid: "5beeefe8bef54caeac463d05d8c30fa6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: UNKNOWN } } }
    #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (libkudu_cli_actions.so+0x28bfbc)
    #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_cli_actions.so+0x28bf2c)
    #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_cli_actions.so+0x28ab95)
    #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x14082f)
    #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132da1)
    #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:270:10 (libkudu_tools_util.so+0x130a52)
    #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:114:15 (kudu+0x324910)
    #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:186:16 (kudu+0x325247)
    #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:249:10 (kudu+0x325671)

  Thread T144 'apply [worker]-' (tid=1286, finished) created by thread T82 at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x43f5cb)
    #4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x43d8cb)
    #5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x43fb32)
    #6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:23 (libtablet.so+0x448cf0)
    #7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:476:5 (libtablet.so+0x449289)
    #8 kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:860:15 (libtablet.so+0x41362a)
I20261120 03:00:53.263901  1324 tablet_bootstrap.cc:492] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:53.264331  1324 ts_tablet_manager.cc:1449] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.094s	user 0.009s	sys 0.000s
I20261120 03:00:53.266639  1324 raft_consensus.cc:366] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.267928  1324 raft_consensus.cc:392] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:53.268535  1324 raft_consensus.cc:787] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:53.248934   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet a72d5d7896de4020a1ac4d2f8c551e15 (table default.Table_f601bffe6ddb49b3a3c589ac7b25235c [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]) successfully deleted
I20261120 03:00:53.271673   498 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_5c5911c8746d4b26843ec77c8141d3b6" } modify_external_catalogs: true
I20261120 03:00:53.272184   498 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_5c5911c8746d4b26843ec77c8141d3b6" } modify_external_catalogs: true
    #9 decltype(std::__1::forward<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x4135b8)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x413538)
I20261120 03:00:53.263469  1392 ts_tablet_manager.cc:1962] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
    #11 std::__1::__function::__alloc_func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x4134f3)
    #12 std::__1::__function::__func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41270c)
    #13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x40fe06)
W20261120 03:00:53.269332   486 catalog_manager.cc:5061] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet df61317717d3401eb74cd93aa336c4dc because tablet deleting was already in progress. No further retry: Already present: State transition of tablet df61317717d3401eb74cd93aa336c4dc already in progress: creating tablet
    #14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x40a5b8)
    #15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3598:3 (libconsensus.so+0x1fd9c0)
    #16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:200:12 (libconsensus.so+0x1d2e9d)
    #17 kudu::consensus::RaftConsensus::UpdateReplica(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::consensus::RaftConsensus::FlushContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1603:5 (libconsensus.so+0x1f0713)
    #18 kudu::consensus::RaftConsensus::Update(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1162:9 (libconsensus.so+0x1efd04)
    #19 kudu::tserver::ConsensusServiceImpl::UpdateConsensus(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1792:25 (libtserver.so+0x26e199)
    #20 kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/consensus/consensus.service.cc:321:13 (libconsensus_proto.so+0xd8d34)
    #21 decltype(std::__1::forward<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus_proto.so+0xd8cc2)
    #22 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus_proto.so+0xd8bf1)
    #23 std::__1::__function::__alloc_func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus_proto.so+0xd8b6c)
    #24 std::__1::__function::__func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus_proto.so+0xd7e22)
    #25 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x21b0bc)
    #26 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x21a4f6)
    #27 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x219e9f)
    #28 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15 (libkrpc.so+0x21d16f)
    #29 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5 (libkrpc.so+0x21e4c1)
    #30 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x21e479)
    #31 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x21e409)
    #32 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x21e3d1)
    #33 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x21d6cd)
    #34 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #35 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #36 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)


CatalogManagerConfigurations/MasterStressTest.Test/0: WARNING: ThreadSanitizer: data race (pid=977)  Write of size 8 at 0x7b44000b4e38 by thread T127:
    #0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (kudu+0x321819)
    #1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (kudu+0x3287c9)
    #2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (kudu+0x328769)
    #3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x2aebd2)
    #4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x2aeb69)
    #5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x381bcd)
    #6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x381500)
    #7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
    #8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libkudu_cli_actions.so+0x24d1d5)
    #9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x450864)
    #11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x403c45)
    #12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x403e59)
    #13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libkudu_cli_actions.so+0x2fbea7)
    #14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libkudu_cli_actions.so+0x2fbe69)
    #15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libkudu_cli_actions.so+0x2fbe29)
    #16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libkudu_cli_actions.so+0x2fa1fa)
    #17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3e1f09)
    #18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3e1ee0)
    #19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3e1ea0)
    #20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b4aa6)
    #21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b49c4)
    #22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b477b)
    #23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x28c761)
    #24 kudu::tserver::TSTabletManager::PopulateIncrementalTabletReport(kudu::master::TabletReportPB*, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, bool) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1893:1 (libtserver.so+0x2c1b50)
    #25 kudu::tserver::Heartbeater::Thread::GenerateIncrementalTabletReport(kudu::master::TabletReportPB*, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:790:30 (libtserver.so+0x1cb58a)
    #26 kudu::tserver::Heartbeater::Thread::DoHeartbeat(kudu::master::MasterErrorPB*, kudu::rpc::ErrorStatusPB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:521:5 (libtserver.so+0x1cd4c4)
    #27 kudu::tserver::Heartbeater::Thread::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:645:21 (libtserver.so+0x1ce790)
    #28 kudu::tserver::Heartbeater::Thread::Start()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:728:48 (libtserver.so+0x1d03c1)
    #29 decltype(std::__1::forward<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(fp)()) std::__1::__invoke<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x1d0379)
    #30 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x1d0309)
    #31 std::__1::__function::__alloc_func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x1d02d1)
    #32 std::__1::__function::__func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x1cf5cd)
    #33 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #34 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #35 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Previous atomic write of size 8 at 0x7b44000b4e38 by thread T144 (mutexes: write M299906313465758112):
    #0 __tsan_atomic64_compare_exchange_val /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:850 (kudu+0x2d19c8)
    #1 bool std::__1::__cxx_atomic_compare_exchange_weak<long>(std::__1::__cxx_atomic_base_impl<long>*, long*, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1031:12 (liblog.so+0x8c014)
    #2 std::__1::__atomic_base<long, false>::compare_exchange_weak(long&, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1622:17 (liblog.so+0x8be39)
    #3 void kudu::AtomicStoreMax<long>(std::__1::atomic<long>&, long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/atomic-utils.h:33:13 (liblog.so+0x7ec80)
    #4 kudu::HighWaterMark::UpdateMax(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:80:5 (libkudu_util.so+0x380794)
    #5 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:5 (libkudu_util.so+0x37b5fd)
    #6 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x378c61)
    #7 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x451230)
    #8 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:608:16 (libtablet.so+0x44a1c2)
    #9 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:579:5 (libtablet.so+0x449953)
    #10 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:65 (libtablet.so+0x44db61)
    #11 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x44db19)
    #12 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x44daa9)
    #13 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x44da71)
    #14 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x44cc9d)
    #15 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
I20261120 03:00:53.411835  1295 ts_tablet_manager.cc:1962] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
    #16 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #17 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x4403a5)
    #18 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x442711)
    #19 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x4426c9)
    #20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x442659)
    #21 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x442621)
    #22 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x44191d)
    #23 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #24 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #25 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Mutex M299906313465758112 is already destroyed.

  Thread T127 'heartbeat-1124' (tid=1124, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
I20261120 03:00:53.418100   710 leader_election.cc:304] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::tserver::Heartbeater::Thread::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:727:10 (libtserver.so+0x1cae61)
    #4 kudu::tserver::Heartbeater::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:251:41 (libtserver.so+0x1cac3d)
I20261120 03:00:53.419337  1289 raft_consensus.cc:3024] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
    #5 kudu::tserver::TabletServer::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:167:3 (libtserver.so+0x259ccc)
    #6 kudu::tserver::RunTabletServer() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server_runner.cc:86:3 (libtserver.so+0x266e93)
    #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:128:10 (libkudu_cli_actions.so+0x3b6429)
I20261120 03:00:53.419744  1289 raft_consensus.cc:534] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
    #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_cli_actions.so+0x28c051)
I20261120 03:00:53.420060  1289 raft_consensus.cc:3296] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
    #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (libkudu_cli_actions.so+0x28bfbc)
    #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_cli_actions.so+0x28bf2c)
    #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_cli_actions.so+0x28ab95)
    #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x14082f)
    #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132da1)
    #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:270:10 (libkudu_tools_util.so+0x130a52)
    #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:114:15 (kudu+0x324910)
    #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:186:16 (kudu+0x325247)
    #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:249:10 (kudu+0x325671)

  Thread T144 'apply [worker]-' (tid=1286, finished) created by thread T82 at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x43f5cb)
    #4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x43d8cb)
    #5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x43fb32)
    #6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:23 (libtablet.so+0x448cf0)
    #7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:476:5 (libtablet.so+0x449289)
    #8 kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:860:15 (libtablet.so+0x41362a)
    #9 decltype(std::__1::forward<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x4135b8)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x413538)
    #11 std::__1::__function::__alloc_func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x4134f3)
    #12 std::__1::__function::__func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41270c)
    #13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x40fe06)
    #14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x40a5b8)
    #15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3598:3 (libconsensus.so+0x1fd9c0)
    #16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:200:12 (libconsensus.so+0x1d2e9d)
    #17 kudu::consensus::RaftConsensus::UpdateReplica(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::consensus::RaftConsensus::FlushContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1603:5 (libconsensus.so+0x1f0713)
    #18 kudu::consensus::RaftConsensus::Update(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1162:9 (libconsensus.so+0x1efd04)
    #19 kudu::tserver::ConsensusServiceImpl::UpdateConsensus(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1792:25 (libtserver.so+0x26e199)
    #20 kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/consensus/consensus.service.cc:321:13 (libconsensus_proto.so+0xd8d34)
    #21 decltype(std::__1::forward<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus_proto.so+0xd8cc2)
    #22 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus_proto.so+0xd8bf1)
    #23 std::__1::__function::__alloc_func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus_proto.so+0xd8b6c)
    #24 std::__1::__function::__func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus_proto.so+0xd7e22)
    #25 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x21b0bc)
    #26 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x21a4f6)
    #27 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x219e9f)
    #28 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15 (libkrpc.so+0x21d16f)
    #29 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5 (libkrpc.so+0x21e4c1)
    #30 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x21e479)
    #31 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x21e409)
    #32 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x21e3d1)
    #33 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x21d6cd)
    #34 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #35 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #36 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)


CatalogManagerConfigurations/MasterStressTest.Test/0: WARNING: ThreadSanitizer: data race (pid=977)  Write of size 8 at 0x7b44000b4e40 by thread T127:
    #0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (kudu+0x321819)
    #1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (kudu+0x3287c9)
    #2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (kudu+0x328769)
    #3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x2aebd2)
    #4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x2aeb69)
    #5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x381bcd)
    #6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x381500)
    #7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
    #8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libkudu_cli_actions.so+0x24d1d5)
    #9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x450864)
    #11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x403c45)
    #12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x403e59)
    #13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libkudu_cli_actions.so+0x2fbea7)
    #14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libkudu_cli_actions.so+0x2fbe69)
    #15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libkudu_cli_actions.so+0x2fbe29)
    #16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libkudu_cli_actions.so+0x2fa1fa)
    #17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3e1f09)
    #18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3e1ee0)
    #19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3e1ea0)
    #20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b4aa6)
    #21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b49c4)
    #22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b477b)
    #23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x28c761)
    #24 kudu::tserver::TSTabletManager::PopulateIncrementalTabletReport(kudu::master::TabletReportPB*, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, bool) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1893:1 (libtserver.so+0x2c1b50)
    #25 kudu::tserver::Heartbeater::Thread::GenerateIncrementalTabletReport(kudu::master::TabletReportPB*, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:790:30 (libtserver.so+0x1cb58a)
    #26 kudu::tserver::Heartbeater::Thread::DoHeartbeat(kudu::master::MasterErrorPB*, kudu::rpc::ErrorStatusPB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:521:5 (libtserver.so+0x1cd4c4)
    #27 kudu::tserver::Heartbeater::Thread::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:645:21 (libtserver.so+0x1ce790)
    #28 kudu::tserver::Heartbeater::Thread::Start()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:728:48 (libtserver.so+0x1d03c1)
    #29 decltype(std::__1::forward<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(fp)()) std::__1::__invoke<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x1d0379)
    #30 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x1d0309)
    #31 std::__1::__function::__alloc_func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x1d02d1)
    #32 std::__1::__function::__func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x1cf5cd)
    #33 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #34 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #35 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Previous read of size 8 at 0x7b44000b4e40 by thread T144 (mutexes: write M299906313465758112):
    #0 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::begin() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1520:30 (libkudu_util.so+0x37b4f9)
    #1 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:284:22 (libkudu_util.so+0x378c0a)
    #2 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x451230)
    #3 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:608:16 (libtablet.so+0x44a1c2)
    #4 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:579:5 (libtablet.so+0x449953)
    #5 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:65 (libtablet.so+0x44db61)
    #6 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x44db19)
    #7 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x44daa9)
    #8 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x44da71)
    #9 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x44cc9d)
    #10 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #11 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #12 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x4403a5)
    #13 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x442711)
    #14 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x4426c9)
    #15 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x442659)
    #16 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x442621)
    #17 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x44191d)
    #18 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #19 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #20 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Mutex M299906313465758112 is already destroyed.

  Thread T127 'heartbeat-1124' (tid=1124, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::tserver::Heartbeater::Thread::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:727:10 (libtserver.so+0x1cae61)
    #4 kudu::tserver::Heartbeater::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:251:41 (libtserver.so+0x1cac3d)
    #5 kudu::tserver::TabletServer::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:167:3 (libtserver.so+0x259ccc)
    #6 kudu::tserver::RunTabletServer() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server_runner.cc:86:3 (libtserver.so+0x266e93)
    #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:128:10 (libkudu_cli_actions.so+0x3b6429)
    #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_cli_actions.so+0x28c051)
    #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (libkudu_cli_actions.so+0x28bfbc)
    #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_cli_actions.so+0x28bf2c)
    #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_cli_actions.so+0x28ab95)
    #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x14082f)
    #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132da1)
    #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:270:10 (libkudu_tools_util.so+0x130a52)
    #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:114:15 (kudu+0x324910)
    #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:186:16 (kudu+0x325247)
    #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:249:10 (kudu+0x325671)

  Thread T144 'apply [worker]-' (tid=1286, finished) created by thread T82 at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x43f5cb)
    #4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x43d8cb)
    #5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x43fb32)
    #6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:23 (libtablet.so+0x448cf0)
    #7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:476:5 (libtablet.so+0x449289)
    #8 kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:860:15 (libtablet.so+0x41362a)
    #9 decltype(std::__1::forward<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x4135b8)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x413538)
    #11 std::__1::__function::__alloc_func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x4134f3)
    #12 std::__1::__function::__func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41270c)
    #13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x40fe06)
    #14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x40a5b8)
    #15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3598:3 (libconsensus.so+0x1fd9c0)
    #16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:200:12 (libconsensus.so+0x1d2e9d)
    #17 kudu::consensus::RaftConsensus::UpdateReplica(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::consensus::RaftConsensus::FlushContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1603:5 (libconsensus.so+0x1f0713)
    #18 kudu::consensus::RaftConsensus::Update(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1162:9 (libconsensus.so+0x1efd04)
    #19 kudu::tserver::ConsensusServiceImpl::UpdateConsensus(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1792:25 (libtserver.so+0x26e199)
    #20 kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/consensus/consensus.service.cc:321:13 (libconsensus_proto.so+0xd8d34)
    #21 decltype(std::__1::forward<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus_proto.so+0xd8cc2)
    #22 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus_proto.so+0xd8bf1)
    #23 std::__1::__function::__alloc_func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus_proto.so+0xd8b6c)
    #24 std::__1::__function::__func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus_proto.so+0xd7e22)
    #25 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x21b0bc)
    #26 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x21a4f6)
    #27 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x219e9f)
    #28 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15 (libkrpc.so+0x21d16f)
    #29 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5 (libkrpc.so+0x21e4c1)
    #30 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x21e479)
    #31 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x21e409)
    #32 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x21e3d1)
    #33 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x21d6cd)
I20261120 03:00:53.640141  1046 tablet_service.cc:1495] Processing CreateTablet for tablet a08aa5ddcf1a4061a70275a8eecb1856 (DEFAULT_TABLE table=default.Table_3c79e29085ad4875874397eaefcab250 [id=47e54c07e54d40adb2ac6907cb03bb68]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.641237  1046 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a08aa5ddcf1a4061a70275a8eecb1856. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.642251  1280 raft_consensus.cc:534] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:53.642604  1280 raft_consensus.cc:3296] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:53.653661  1280 raft_consensus.cc:556] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.655057   899 tablet_service.cc:1495] Processing CreateTablet for tablet a08aa5ddcf1a4061a70275a8eecb1856 (DEFAULT_TABLE table=default.Table_3c79e29085ad4875874397eaefcab250 [id=47e54c07e54d40adb2ac6907cb03bb68]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.656447   899 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a08aa5ddcf1a4061a70275a8eecb1856. 1 dirs total, 0 dirs full, 0 dirs failed
    #34 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #35 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #36 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)
I20261120 03:00:53.605322  1405 consensus_queue.cc:1049] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s


CatalogManagerConfigurations/MasterStressTest.Test/3: Unrecognized error type. Please see the error log for more information.

Full log

Note: This is test shard 1 of 3.
[==========] Running 2 tests from 1 test suite.
[----------] Global test environment set-up.
[----------] 2 tests from CatalogManagerConfigurations/MasterStressTest
[ RUN      ] CatalogManagerConfigurations/MasterStressTest.Test/0
2026-11-20T03:00:32Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
2026-11-20T03:00:32Z Disabled control of system clock
WARNING: Logging before InitGoogleLogging() is written to STDERR
I20261120 03:00:32.829859   385 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
/tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-0/wal
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-0/logs
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.126:44467
--webserver_interface=127.0.96.126
--webserver_port=0
--builtin_ntp_servers=127.0.96.84:35145
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.96.126:44467,127.0.96.125:37411,127.0.96.124:38133
--hive_metastore_notification_log_poll_period_seconds=1
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
W20261120 03:00:33.136698   394 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
W20261120 03:00:33.137269   394 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20261120 03:00:33.137661   394 flags.cc:432] Enabled unsafe flag: --never_fsync=true
W20261120 03:00:33.171001   394 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
W20261120 03:00:33.171320   394 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
W20261120 03:00:33.171536   394 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
W20261120 03:00:33.171743   394 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
I20261120 03:00:33.209364   394 master_runner.cc:387] Master server non-default flags:
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-0/logs
--logbuflevel=-1
--logtostderr=true
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.96.84:35145
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-0/data
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-0/wal
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.96.126:44467,127.0.96.125:37411,127.0.96.124:38133
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.126:44467
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-0/data/info.pb
--webserver_interface=127.0.96.126
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false

Master server version:
kudu 1.19.0-SNAPSHOT
revision 36b6772d828145f8fbba3c49746c86838683f9ba
build type FASTDEBUG
built by None at 20 Nov 2026 02:45:53 UTC on bdcb31816ec0
build id 14870
TSAN enabled
I20261120 03:00:33.210711   394 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20261120 03:00:33.212388   394 file_cache.cc:504] Constructed file cache file cache with capacity 419430
W20261120 03:00:33.224237   400 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:00:34.627012   399 debug-util.cc:398] Leaking SignalData structure 0x7b08000382a0 after lost signal to thread 394
W20261120 03:00:34.794499   394 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.570s	user 0.719s	sys 0.850s
W20261120 03:00:33.224498   401 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:00:34.794859   394 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.571s	user 0.720s	sys 0.850s
W20261120 03:00:34.796820   403 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:00:34.801465   402 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1575 milliseconds
I20261120 03:00:34.801532   394 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20261120 03:00:34.802778   394 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20261120 03:00:34.805478   394 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20261120 03:00:34.806816   394 hybrid_clock.cc:648] HybridClock initialized: now 1795143634806753 us; error 72 us; skew 500 ppm
I20261120 03:00:34.807641   394 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20261120 03:00:34.813750   394 webserver.cc:577] Webserver started at http://127.0.96.126:45831/ using document root <none> and password file <none>
I20261120 03:00:34.814692   394 fs_manager.cc:362] Metadata directory not provided
I20261120 03:00:34.814885   394 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20261120 03:00:34.815330   394 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
I20261120 03:00:34.819615   394 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-0/data/instance:
uuid: "e102e07459954b4ea13b65ab9fd5d18b"
format_stamp: "Formatted at 2026-11-20 03:00:34 on dist-test-slave-jpwh"
I20261120 03:00:34.820731   394 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-0/wal/instance:
uuid: "e102e07459954b4ea13b65ab9fd5d18b"
format_stamp: "Formatted at 2026-11-20 03:00:34 on dist-test-slave-jpwh"
I20261120 03:00:34.827970   394 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.008s	sys 0.000s
I20261120 03:00:34.834060   410 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:00:34.835269   394 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.002s
I20261120 03:00:34.835633   394 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-0/data,/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-0/wal
uuid: "e102e07459954b4ea13b65ab9fd5d18b"
format_stamp: "Formatted at 2026-11-20 03:00:34 on dist-test-slave-jpwh"
I20261120 03:00:34.835958   394 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-0/wal
metadata directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-0/wal
1 data directories: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20261120 03:00:34.893989   394 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20261120 03:00:34.895479   394 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20261120 03:00:34.895907   394 kserver.cc:163] Server-wide thread pool size limit: 3276
I20261120 03:00:34.967357   394 rpc_server.cc:307] RPC server started. Bound to: 127.0.96.126:44467
I20261120 03:00:34.967424   461 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.96.126:44467 every 8 connection(s)
I20261120 03:00:34.970127   394 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-0/data/info.pb
I20261120 03:00:34.976792   462 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:34.978329   385 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu as pid 394
I20261120 03:00:34.978741   385 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-0/wal/instance
I20261120 03:00:34.982228   385 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
/tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-1/wal
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-1/logs
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.125:37411
--webserver_interface=127.0.96.125
--webserver_port=0
--builtin_ntp_servers=127.0.96.84:35145
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.96.126:44467,127.0.96.125:37411,127.0.96.124:38133
--hive_metastore_notification_log_poll_period_seconds=1
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
I20261120 03:00:34.987918   462 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } has no permanent_uuid. Determining permanent_uuid...
I20261120 03:00:35.022389   462 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } has no permanent_uuid. Determining permanent_uuid...
W20261120 03:00:35.024863   412 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.125:37411: connect: Connection refused (error 111)
W20261120 03:00:35.029374   462 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.125:37411: Network error: Client connection negotiation failed: client connection to 127.0.96.125:37411: connect: Connection refused (error 111)
I20261120 03:00:35.078949   462 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } attempt: 1
W20261120 03:00:35.083144   462 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.125:37411: Network error: Client connection negotiation failed: client connection to 127.0.96.125:37411: connect: Connection refused (error 111)
I20261120 03:00:35.151522   462 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } attempt: 2
W20261120 03:00:35.155551   462 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.125:37411: Network error: Client connection negotiation failed: client connection to 127.0.96.125:37411: connect: Connection refused (error 111)
I20261120 03:00:35.246977   462 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } attempt: 3
W20261120 03:00:35.251581   462 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.125:37411: Network error: Client connection negotiation failed: client connection to 127.0.96.125:37411: connect: Connection refused (error 111)
W20261120 03:00:35.302727   464 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
W20261120 03:00:35.303270   464 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20261120 03:00:35.303668   464 flags.cc:432] Enabled unsafe flag: --never_fsync=true
W20261120 03:00:35.339738   464 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
W20261120 03:00:35.340018   464 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
W20261120 03:00:35.340229   464 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
W20261120 03:00:35.340440   464 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
I20261120 03:00:35.377884   464 master_runner.cc:387] Master server non-default flags:
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-1/logs
--logbuflevel=-1
--logtostderr=true
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.96.84:35145
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-1/data
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-1/wal
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.96.126:44467,127.0.96.125:37411,127.0.96.124:38133
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.125:37411
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-1/data/info.pb
--webserver_interface=127.0.96.125
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false

Master server version:
kudu 1.19.0-SNAPSHOT
revision 36b6772d828145f8fbba3c49746c86838683f9ba
build type FASTDEBUG
built by None at 20 Nov 2026 02:45:53 UTC on bdcb31816ec0
build id 14870
TSAN enabled
I20261120 03:00:35.379247   464 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20261120 03:00:35.380776   464 file_cache.cc:504] Constructed file cache file cache with capacity 419430
W20261120 03:00:35.392006   473 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:00:35.395015   462 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } attempt: 4
W20261120 03:00:35.400805   462 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.125:37411: Network error: Client connection negotiation failed: client connection to 127.0.96.125:37411: connect: Connection refused (error 111)
I20261120 03:00:35.700224   462 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } attempt: 5
W20261120 03:00:35.706990   462 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.125:37411: Network error: Client connection negotiation failed: client connection to 127.0.96.125:37411: connect: Connection refused (error 111)
I20261120 03:00:36.254482   462 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } attempt: 6
W20261120 03:00:36.283835   462 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.125:37411: Network error: Client connection negotiation failed: client connection to 127.0.96.125:37411: connect: Connection refused (error 111)
W20261120 03:00:35.392771   474 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:00:36.693298   476 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:00:36.695740   475 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1298 milliseconds
W20261120 03:00:36.696349   464 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.304s	user 0.521s	sys 0.780s
W20261120 03:00:36.696619   464 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.304s	user 0.521s	sys 0.780s
I20261120 03:00:36.696859   464 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20261120 03:00:36.697902   464 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20261120 03:00:36.700436   464 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20261120 03:00:36.701807   464 hybrid_clock.cc:648] HybridClock initialized: now 1795143636701757 us; error 64 us; skew 500 ppm
I20261120 03:00:36.702757   464 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20261120 03:00:36.710601   464 webserver.cc:577] Webserver started at http://127.0.96.125:36919/ using document root <none> and password file <none>
I20261120 03:00:36.711524   464 fs_manager.cc:362] Metadata directory not provided
I20261120 03:00:36.711735   464 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20261120 03:00:36.712165   464 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
I20261120 03:00:36.716439   464 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-1/data/instance:
uuid: "18331bef5c884dcab9dbd26bc2c82273"
format_stamp: "Formatted at 2026-11-20 03:00:36 on dist-test-slave-jpwh"
I20261120 03:00:36.717655   464 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-1/wal/instance:
uuid: "18331bef5c884dcab9dbd26bc2c82273"
format_stamp: "Formatted at 2026-11-20 03:00:36 on dist-test-slave-jpwh"
I20261120 03:00:36.725930   464 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.006s	sys 0.000s
I20261120 03:00:36.732101   485 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:00:36.733285   464 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.001s
I20261120 03:00:36.733597   464 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-1/data,/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-1/wal
uuid: "18331bef5c884dcab9dbd26bc2c82273"
format_stamp: "Formatted at 2026-11-20 03:00:36 on dist-test-slave-jpwh"
I20261120 03:00:36.733914   464 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-1/wal
metadata directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-1/wal
1 data directories: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20261120 03:00:36.792411   464 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20261120 03:00:36.793828   464 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20261120 03:00:36.794269   464 kserver.cc:163] Server-wide thread pool size limit: 3276
I20261120 03:00:36.881923   464 rpc_server.cc:307] RPC server started. Bound to: 127.0.96.125:37411
I20261120 03:00:36.881984   536 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.96.125:37411 every 8 connection(s)
I20261120 03:00:36.884642   464 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-1/data/info.pb
I20261120 03:00:36.890637   537 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:36.890836   385 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu as pid 464
I20261120 03:00:36.891199   385 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-1/wal/instance
I20261120 03:00:36.893226   385 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
/tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/wal
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/logs
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.124:38133
--webserver_interface=127.0.96.124
--webserver_port=0
--builtin_ntp_servers=127.0.96.84:35145
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.96.126:44467,127.0.96.125:37411,127.0.96.124:38133
--hive_metastore_notification_log_poll_period_seconds=1
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
I20261120 03:00:36.903066   537 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } has no permanent_uuid. Determining permanent_uuid...
I20261120 03:00:36.927052   537 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } has no permanent_uuid. Determining permanent_uuid...
I20261120 03:00:36.941135   537 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } has no permanent_uuid. Determining permanent_uuid...
W20261120 03:00:36.943625   486 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
W20261120 03:00:36.948308   537 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:38133: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
I20261120 03:00:36.997805   537 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } attempt: 1
W20261120 03:00:37.002015   537 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:38133: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
I20261120 03:00:37.070411   537 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } attempt: 2
W20261120 03:00:37.074944   537 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:38133: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
I20261120 03:00:37.166326   537 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } attempt: 3
W20261120 03:00:37.170403   537 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:38133: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
W20261120 03:00:37.212309   539 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
W20261120 03:00:37.212879   539 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20261120 03:00:37.213320   539 flags.cc:432] Enabled unsafe flag: --never_fsync=true
W20261120 03:00:37.246641   539 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
W20261120 03:00:37.246966   539 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
W20261120 03:00:37.247222   539 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
W20261120 03:00:37.247464   539 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
I20261120 03:00:37.286587   539 master_runner.cc:387] Master server non-default flags:
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/logs
--logbuflevel=-1
--logtostderr=true
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.96.84:35145
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/data
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/wal
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.96.126:44467,127.0.96.125:37411,127.0.96.124:38133
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.124:38133
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/data/info.pb
--webserver_interface=127.0.96.124
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false

Master server version:
kudu 1.19.0-SNAPSHOT
revision 36b6772d828145f8fbba3c49746c86838683f9ba
build type FASTDEBUG
built by None at 20 Nov 2026 02:45:53 UTC on bdcb31816ec0
build id 14870
TSAN enabled
I20261120 03:00:37.287957   539 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20261120 03:00:37.289551   539 file_cache.cc:504] Constructed file cache file cache with capacity 419430
W20261120 03:00:37.300609   549 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:00:37.313783   537 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } attempt: 4
W20261120 03:00:37.318985   537 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:38133: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
I20261120 03:00:37.344355   462 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } attempt: 7
I20261120 03:00:37.363886   462 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } has no permanent_uuid. Determining permanent_uuid...
W20261120 03:00:37.371857   462 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:38133: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
I20261120 03:00:37.430346   462 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } attempt: 1
W20261120 03:00:37.436448   462 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:38133: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
I20261120 03:00:37.517951   462 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } attempt: 2
W20261120 03:00:37.523940   462 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:38133: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
I20261120 03:00:37.609447   462 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } attempt: 3
W20261120 03:00:37.615979   462 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:38133: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
I20261120 03:00:37.618462   537 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } attempt: 5
W20261120 03:00:37.623710   537 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:38133: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
I20261120 03:00:37.756492   462 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } attempt: 4
W20261120 03:00:37.761829   462 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:38133: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
I20261120 03:00:38.045415   462 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } attempt: 5
W20261120 03:00:38.051748   462 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:38133: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
I20261120 03:00:38.171203   537 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } attempt: 6
W20261120 03:00:38.180115   537 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:38133: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
W20261120 03:00:37.301688   550 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:00:37.305846   552 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:00:38.444343   551 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1139 milliseconds
I20261120 03:00:38.444458   539 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20261120 03:00:38.445639   539 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20261120 03:00:38.448573   539 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20261120 03:00:38.449980   539 hybrid_clock.cc:648] HybridClock initialized: now 1795143638449954 us; error 47 us; skew 500 ppm
I20261120 03:00:38.450945   539 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20261120 03:00:38.457455   539 webserver.cc:577] Webserver started at http://127.0.96.124:43247/ using document root <none> and password file <none>
I20261120 03:00:38.458400   539 fs_manager.cc:362] Metadata directory not provided
I20261120 03:00:38.458601   539 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20261120 03:00:38.459066   539 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
I20261120 03:00:38.463371   539 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/data/instance:
uuid: "2599fd9eef7d4ce990c40e2356a37c3f"
format_stamp: "Formatted at 2026-11-20 03:00:38 on dist-test-slave-jpwh"
I20261120 03:00:38.464476   539 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/wal/instance:
uuid: "2599fd9eef7d4ce990c40e2356a37c3f"
format_stamp: "Formatted at 2026-11-20 03:00:38 on dist-test-slave-jpwh"
I20261120 03:00:38.471701   539 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.000s	sys 0.008s
I20261120 03:00:38.477519   563 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:00:38.478654   539 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.002s
I20261120 03:00:38.478960   539 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/data,/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/wal
uuid: "2599fd9eef7d4ce990c40e2356a37c3f"
format_stamp: "Formatted at 2026-11-20 03:00:38 on dist-test-slave-jpwh"
I20261120 03:00:38.479286   539 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/wal
metadata directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/wal
1 data directories: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20261120 03:00:38.542639   539 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20261120 03:00:38.544121   539 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20261120 03:00:38.544535   539 kserver.cc:163] Server-wide thread pool size limit: 3276
I20261120 03:00:38.604226   462 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } attempt: 6
W20261120 03:00:38.611760   462 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:38133: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
I20261120 03:00:38.622349   539 rpc_server.cc:307] RPC server started. Bound to: 127.0.96.124:38133
I20261120 03:00:38.622414   616 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.96.124:38133 every 8 connection(s)
I20261120 03:00:38.625027   539 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/data/info.pb
I20261120 03:00:38.629420   385 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu as pid 539
I20261120 03:00:38.629782   385 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/wal/instance
I20261120 03:00:38.631424   617 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:38.647066   617 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } has no permanent_uuid. Determining permanent_uuid...
I20261120 03:00:38.674032   617 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } has no permanent_uuid. Determining permanent_uuid...
I20261120 03:00:38.686177   617 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } has no permanent_uuid. Determining permanent_uuid...
I20261120 03:00:38.704507   617 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f: Bootstrap starting.
I20261120 03:00:38.712126   617 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:38.714210   617 log.cc:826] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f: Log is configured to *not* fsync() on all Append() calls
I20261120 03:00:38.719470   617 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f: No bootstrap required, opened a new log
I20261120 03:00:38.739490   617 raft_consensus.cc:366] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:38.740114   617 raft_consensus.cc:392] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:38.740372   617 raft_consensus.cc:787] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2599fd9eef7d4ce990c40e2356a37c3f, State: Initialized, Role: FOLLOWER
I20261120 03:00:38.741086   617 consensus_queue.cc:261] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:38.743786   628 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:38.744359   628 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:00:38.744887   617 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: configured and running, proceeding with master startup.
W20261120 03:00:38.772679   639 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:00:38.773116   639 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:00:38.766619   539 master_runner.cc:187] Error getting master registration for 127.0.96.126:44467: OK, instance_id { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" instance_seqno: 1795143634948248 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
I20261120 03:00:38.789217   539 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:00:38.800112   569 tablet.cc:2554] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20261120 03:00:38.865656   628 raft_consensus.cc:534] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:38.866276   628 raft_consensus.cc:556] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:38.870724   628 leader_election.cc:290] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e102e07459954b4ea13b65ab9fd5d18b (127.0.96.126:44467), 18331bef5c884dcab9dbd26bc2c82273 (127.0.96.125:37411)
I20261120 03:00:39.240598   537 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } attempt: 7
I20261120 03:00:39.274194   537 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Bootstrap starting.
I20261120 03:00:39.282270   537 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:39.284523   537 log.cc:826] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Log is configured to *not* fsync() on all Append() calls
I20261120 03:00:39.289963   537 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: No bootstrap required, opened a new log
I20261120 03:00:39.317197   537 raft_consensus.cc:366] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:39.318051   537 raft_consensus.cc:392] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:39.318321   537 raft_consensus.cc:787] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 18331bef5c884dcab9dbd26bc2c82273, State: Initialized, Role: FOLLOWER
I20261120 03:00:39.319079   537 consensus_queue.cc:261] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:39.321902   644 sys_catalog.cc:455] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:39.322629   644 sys_catalog.cc:458] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:00:39.323367   537 sys_catalog.cc:565] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [sys.catalog]: configured and running, proceeding with master startup.
I20261120 03:00:39.323686   512 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "18331bef5c884dcab9dbd26bc2c82273" is_pre_election: true
I20261120 03:00:39.324729   512 raft_consensus.cc:2062] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 2599fd9eef7d4ce990c40e2356a37c3f in term 0
I20261120 03:00:39.326232   565 leader_election.cc:304] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 18331bef5c884dcab9dbd26bc2c82273, 2599fd9eef7d4ce990c40e2356a37c3f; no voters: 
I20261120 03:00:39.327186   628 raft_consensus.cc:3024] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:39.327509   628 raft_consensus.cc:534] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:39.334304   628 raft_consensus.cc:3296] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:39.350281   628 raft_consensus.cc:556] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:39.357224   628 leader_election.cc:290] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [CANDIDATE]: Term 1 election: Requested vote from peers e102e07459954b4ea13b65ab9fd5d18b (127.0.96.126:44467), 18331bef5c884dcab9dbd26bc2c82273 (127.0.96.125:37411)
I20261120 03:00:39.359208   512 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "18331bef5c884dcab9dbd26bc2c82273"
I20261120 03:00:39.359751   512 raft_consensus.cc:3296] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:39.360884   512 raft_consensus.cc:2062] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 2599fd9eef7d4ce990c40e2356a37c3f in term 1
I20261120 03:00:39.368629   565 leader_election.cc:304] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 18331bef5c884dcab9dbd26bc2c82273, 2599fd9eef7d4ce990c40e2356a37c3f; no voters: 
I20261120 03:00:39.369491   628 raft_consensus.cc:3024] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:39.369987   628 raft_consensus.cc:744] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 1 LEADER]: Becoming Leader. State: Replica: 2599fd9eef7d4ce990c40e2356a37c3f, State: Running, Role: LEADER
I20261120 03:00:39.370913   628 consensus_queue.cc:238] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:39.379199   647 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2599fd9eef7d4ce990c40e2356a37c3f. Latest consensus state: current_term: 1 leader_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:39.379954   647 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: This master's current role is: LEADER
I20261120 03:00:39.385572   652 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
I20261120 03:00:39.389171   647 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [LEADER]: Connected to new peer: Peer: permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20261120 03:00:39.397452   652 catalog_manager.cc:1535] Initializing Kudu cluster ID...
I20261120 03:00:39.409067   644 sys_catalog.cc:455] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2599fd9eef7d4ce990c40e2356a37c3f. Latest consensus state: current_term: 1 leader_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:39.410110   644 sys_catalog.cc:458] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:00:39.413085   653 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:39.413810   653 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: This master's current role is: LEADER
I20261120 03:00:39.420312   644 sys_catalog.cc:455] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:39.421204   644 sys_catalog.cc:458] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:00:39.471007   652 catalog_manager.cc:1398] Generated new cluster ID: 96a176325a9f4b23b7e54922a0ed8e6d
W20261120 03:00:39.470945   662 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:00:39.471288   662 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:00:39.471300   652 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
W20261120 03:00:39.481673   564 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.0.96.126:44467 timed out after 0.611s (SENT)
I20261120 03:00:39.445473   464 master_runner.cc:187] Error getting master registration for 127.0.96.126:44467: OK, instance_id { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" instance_seqno: 1795143634948248 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
I20261120 03:00:39.488924   464 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:00:39.516898   652 catalog_manager.cc:1421] Generated new certificate authority record
I20261120 03:00:39.520159   652 catalog_manager.cc:1555] Loading token signing keys...
I20261120 03:00:39.558517   652 catalog_manager.cc:6237] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f: Generated new TSK 0
I20261120 03:00:39.559953   652 catalog_manager.cc:1565] Initializing in-progress tserver states...
I20261120 03:00:39.587258   385 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
/tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/logs
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.96.65:0
--local_ip_for_outbound_sockets=127.0.96.65
--webserver_interface=127.0.96.65
--webserver_port=0
--tserver_master_addrs=127.0.96.126:44467,127.0.96.125:37411,127.0.96.124:38133
--builtin_ntp_servers=127.0.96.84:35145
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--heartbeat_rpc_timeout_ms=1000
--heartbeat_max_failures_before_backoff=1
--heartbeat_interval_ms=500 with env {}
I20261120 03:00:39.645161   462 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } attempt: 7
I20261120 03:00:39.727033   462 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b: Bootstrap starting.
I20261120 03:00:39.784124   462 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:39.800691   462 log.cc:826] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b: Log is configured to *not* fsync() on all Append() calls
I20261120 03:00:39.841917   462 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b: No bootstrap required, opened a new log
I20261120 03:00:39.931645   462 raft_consensus.cc:366] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:39.935196   462 raft_consensus.cc:392] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:39.937347   462 raft_consensus.cc:787] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e102e07459954b4ea13b65ab9fd5d18b, State: Initialized, Role: FOLLOWER
I20261120 03:00:39.941600   462 consensus_queue.cc:261] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:39.955036   670 sys_catalog.cc:455] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:39.955814   670 sys_catalog.cc:458] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:00:39.964730   462 sys_catalog.cc:565] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [sys.catalog]: configured and running, proceeding with master startup.
I20261120 03:00:39.964892   433 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e102e07459954b4ea13b65ab9fd5d18b"
I20261120 03:00:39.966027   437 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e102e07459954b4ea13b65ab9fd5d18b" is_pre_election: true
W20261120 03:00:39.970476   437 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:54602 (request call id 2) took 1101 ms (client timeout 611 ms). Trace:
W20261120 03:00:39.970922   437 rpcz_store.cc:269] 1120 03:00:38.869353 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:38.869481 (+   128us) service_pool.cc:224] Handling call
1120 03:00:39.970405 (+1100924us) inbound_call.cc:177] Queueing success response
Metrics: {}
I20261120 03:00:39.966974   435 raft_consensus.cc:3296] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:39.967412   436 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e102e07459954b4ea13b65ab9fd5d18b"
I20261120 03:00:39.968062   434 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e102e07459954b4ea13b65ab9fd5d18b" is_pre_election: true
W20261120 03:00:39.983410   434 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:54602 (request call id 5) took 481 ms (client timeout 611 ms). Trace:
W20261120 03:00:39.983852   434 rpcz_store.cc:269] 1120 03:00:39.501539 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:39.501688 (+   149us) service_pool.cc:224] Handling call
1120 03:00:39.983330 (+481642us) inbound_call.cc:177] Queueing success response
Metrics: {}
I20261120 03:00:39.986155   436 raft_consensus.cc:2062] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 2599fd9eef7d4ce990c40e2356a37c3f in term 1
I20261120 03:00:39.993916   647 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [LEADER]: Connected to new peer: Peer: permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20261120 03:00:39.999159   436 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:54602 (request call id 3) took 640 ms (client timeout 526 ms). Trace:
W20261120 03:00:39.999580   436 rpcz_store.cc:269] 1120 03:00:39.358567 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:39.358737 (+   170us) service_pool.cc:224] Handling call
1120 03:00:39.999083 (+640346us) inbound_call.cc:177] Queueing success response
Metrics: {"raft.queue_time_us":6587}
I20261120 03:00:40.045012   674 mvcc.cc:204] Tried to move back new op lower bound from 7352908347084726272 to 7352908346877198336. Current Snapshot: MvccSnapshot[applied={T|T < 7352908347084726272}]
I20261120 03:00:40.078711   670 sys_catalog.cc:455] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2599fd9eef7d4ce990c40e2356a37c3f. Latest consensus state: current_term: 1 leader_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:40.079504   670 sys_catalog.cc:458] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:00:40.078817   672 sys_catalog.cc:455] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:40.080426   672 sys_catalog.cc:458] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:00:40.123488   686 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
W20261120 03:00:40.187435   667 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
W20261120 03:00:40.187997   667 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20261120 03:00:40.188599   667 flags.cc:432] Enabled unsafe flag: --never_fsync=true
W20261120 03:00:40.222625   667 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
W20261120 03:00:40.223618   667 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.96.65
I20261120 03:00:40.262123   667 tablet_server_runner.cc:78] Tablet server non-default flags:
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/logs
--logbuflevel=-1
--logtostderr=true
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.96.84:35145
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/data
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.0.96.65:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/data/info.pb
--webserver_interface=127.0.96.65
--webserver_port=0
--heartbeat_interval_ms=500
--heartbeat_max_failures_before_backoff=1
--heartbeat_rpc_timeout_ms=1000
--tserver_master_addrs=127.0.96.126:44467,127.0.96.125:37411,127.0.96.124:38133
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.0.96.65

Tablet server version:
kudu 1.19.0-SNAPSHOT
revision 36b6772d828145f8fbba3c49746c86838683f9ba
build type FASTDEBUG
built by None at 20 Nov 2026 02:45:53 UTC on bdcb31816ec0
build id 14870
TSAN enabled
I20261120 03:00:40.263491   667 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20261120 03:00:40.265098   667 file_cache.cc:504] Constructed file cache file cache with capacity 419430
W20261120 03:00:40.281211   692 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:00:40.457108   647 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:40.458907   647 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: This master's current role is: LEADER
I20261120 03:00:40.477499   662 catalog_manager.cc:1310] Loaded cluster ID: 96a176325a9f4b23b7e54922a0ed8e6d
I20261120 03:00:40.477819   662 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: loading cluster ID for follower catalog manager: success
I20261120 03:00:40.486918   662 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: acquiring CA information for follower catalog manager: success
I20261120 03:00:40.502960   662 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
I20261120 03:00:40.550052   698 raft_consensus.cc:534] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 2599fd9eef7d4ce990c40e2356a37c3f)
I20261120 03:00:40.552011   698 raft_consensus.cc:556] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:40.560067   435 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "18331bef5c884dcab9dbd26bc2c82273" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "e102e07459954b4ea13b65ab9fd5d18b" is_pre_election: true
I20261120 03:00:40.562546   590 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "18331bef5c884dcab9dbd26bc2c82273" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" is_pre_election: true
I20261120 03:00:40.562713   698 leader_election.cc:290] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e102e07459954b4ea13b65ab9fd5d18b (127.0.96.126:44467), 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133)
I20261120 03:00:40.564534   486 leader_election.cc:304] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 18331bef5c884dcab9dbd26bc2c82273; no voters: 2599fd9eef7d4ce990c40e2356a37c3f, e102e07459954b4ea13b65ab9fd5d18b
I20261120 03:00:40.565373   698 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20261120 03:00:40.976591   699 raft_consensus.cc:534] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 2599fd9eef7d4ce990c40e2356a37c3f)
I20261120 03:00:40.983984   699 raft_consensus.cc:556] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:40.995958   699 leader_election.cc:290] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 18331bef5c884dcab9dbd26bc2c82273 (127.0.96.125:37411), 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133)
I20261120 03:00:40.999063   590 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e102e07459954b4ea13b65ab9fd5d18b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" is_pre_election: true
I20261120 03:00:41.000831   512 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e102e07459954b4ea13b65ab9fd5d18b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "18331bef5c884dcab9dbd26bc2c82273" is_pre_election: true
I20261120 03:00:41.002470   412 leader_election.cc:304] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e102e07459954b4ea13b65ab9fd5d18b; no voters: 18331bef5c884dcab9dbd26bc2c82273, 2599fd9eef7d4ce990c40e2356a37c3f
I20261120 03:00:41.003400   699 raft_consensus.cc:2966] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20261120 03:00:41.150923   686 catalog_manager.cc:1310] Loaded cluster ID: 96a176325a9f4b23b7e54922a0ed8e6d
I20261120 03:00:41.151223   686 catalog_manager.cc:1603] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b: loading cluster ID for follower catalog manager: success
I20261120 03:00:41.178817   686 catalog_manager.cc:1625] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b: acquiring CA information for follower catalog manager: success
I20261120 03:00:41.196075   686 catalog_manager.cc:1653] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
I20261120 03:00:41.690788   701 raft_consensus.cc:534] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 2599fd9eef7d4ce990c40e2356a37c3f)
I20261120 03:00:41.698330   701 raft_consensus.cc:556] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:41.703109   435 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "18331bef5c884dcab9dbd26bc2c82273" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "e102e07459954b4ea13b65ab9fd5d18b" is_pre_election: true
I20261120 03:00:41.705935   701 leader_election.cc:290] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e102e07459954b4ea13b65ab9fd5d18b (127.0.96.126:44467), 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133)
I20261120 03:00:41.706619   590 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "18331bef5c884dcab9dbd26bc2c82273" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" is_pre_election: true
I20261120 03:00:41.708520   486 leader_election.cc:304] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 18331bef5c884dcab9dbd26bc2c82273; no voters: 2599fd9eef7d4ce990c40e2356a37c3f, e102e07459954b4ea13b65ab9fd5d18b
I20261120 03:00:41.709342   701 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20261120 03:00:41.680169   691 debug-util.cc:398] Leaking SignalData structure 0x7b0800032a00 after lost signal to thread 667
W20261120 03:00:42.237460   691 kernel_stack_watchdog.cc:198] Thread 667 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 395ms:
Kernel stack:
(could not read kernel stack)
User stack:
<Timed out: thread did not respond: maybe it is blocking signals>
W20261120 03:00:40.284972   693 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:00:42.238891   667 thread.cc:642] GCE (cloud detector) Time spent creating pthread: real 1.956s	user 0.701s	sys 1.018s
W20261120 03:00:42.239336   667 thread.cc:609] GCE (cloud detector) Time spent starting thread: real 1.957s	user 0.701s	sys 1.018s
W20261120 03:00:42.249192   703 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:00:42.249652   667 server_base.cc:1061] running on GCE node
I20261120 03:00:42.250756   667 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20261120 03:00:42.253060   667 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20261120 03:00:42.254519   667 hybrid_clock.cc:648] HybridClock initialized: now 1795143642254450 us; error 72 us; skew 500 ppm
I20261120 03:00:42.255445   667 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20261120 03:00:42.262207   667 webserver.cc:577] Webserver started at http://127.0.96.65:34321/ using document root <none> and password file <none>
I20261120 03:00:42.263087   667 fs_manager.cc:362] Metadata directory not provided
I20261120 03:00:42.263311   667 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20261120 03:00:42.263772   667 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
I20261120 03:00:42.268009   667 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/data/instance:
uuid: "5beeefe8bef54caeac463d05d8c30fa6"
format_stamp: "Formatted at 2026-11-20 03:00:42 on dist-test-slave-jpwh"
I20261120 03:00:42.269132   667 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/instance:
uuid: "5beeefe8bef54caeac463d05d8c30fa6"
format_stamp: "Formatted at 2026-11-20 03:00:42 on dist-test-slave-jpwh"
I20261120 03:00:42.276968   667 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.009s	sys 0.001s
I20261120 03:00:42.283736   708 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:00:42.284922   667 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.001s	sys 0.003s
I20261120 03:00:42.285236   667 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/data,/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal
uuid: "5beeefe8bef54caeac463d05d8c30fa6"
format_stamp: "Formatted at 2026-11-20 03:00:42 on dist-test-slave-jpwh"
I20261120 03:00:42.285550   667 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal
metadata directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal
1 data directories: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20261120 03:00:42.340176   667 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20261120 03:00:42.342056   667 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20261120 03:00:42.342576   667 kserver.cc:163] Server-wide thread pool size limit: 3276
I20261120 03:00:42.345510   667 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20261120 03:00:42.350809   667 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20261120 03:00:42.351023   667 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:00:42.351286   667 ts_tablet_manager.cc:662] Registered 0 tablets
I20261120 03:00:42.351444   667 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:00:42.385690   728 raft_consensus.cc:534] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 2599fd9eef7d4ce990c40e2356a37c3f)
I20261120 03:00:42.386945   728 raft_consensus.cc:556] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:42.390434   590 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e102e07459954b4ea13b65ab9fd5d18b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" is_pre_election: true
I20261120 03:00:42.390475   512 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e102e07459954b4ea13b65ab9fd5d18b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "18331bef5c884dcab9dbd26bc2c82273" is_pre_election: true
I20261120 03:00:42.392498   728 leader_election.cc:290] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 18331bef5c884dcab9dbd26bc2c82273 (127.0.96.125:37411), 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133)
I20261120 03:00:42.392556   412 leader_election.cc:304] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e102e07459954b4ea13b65ab9fd5d18b; no voters: 18331bef5c884dcab9dbd26bc2c82273, 2599fd9eef7d4ce990c40e2356a37c3f
I20261120 03:00:42.393554   728 raft_consensus.cc:2966] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20261120 03:00:42.558955   667 rpc_server.cc:307] RPC server started. Bound to: 127.0.96.65:39039
I20261120 03:00:42.559013   821 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.96.65:39039 every 8 connection(s)
I20261120 03:00:42.562788   667 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/data/info.pb
I20261120 03:00:42.570822   385 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu as pid 667
I20261120 03:00:42.571333   385 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/instance
I20261120 03:00:42.588352   385 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
/tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/logs
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.96.66:0
--local_ip_for_outbound_sockets=127.0.96.66
--webserver_interface=127.0.96.66
--webserver_port=0
--tserver_master_addrs=127.0.96.126:44467,127.0.96.125:37411,127.0.96.124:38133
--builtin_ntp_servers=127.0.96.84:35145
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--heartbeat_rpc_timeout_ms=1000
--heartbeat_max_failures_before_backoff=1
--heartbeat_interval_ms=500 with env {}
I20261120 03:00:42.627956   822 heartbeater.cc:345] Connected to a master server at 127.0.96.125:37411
I20261120 03:00:42.628494   822 heartbeater.cc:462] Registering TS with master...
I20261120 03:00:42.629820   822 heartbeater.cc:508] Master 127.0.96.125:37411 requested a full tablet report, sending...
I20261120 03:00:42.627949   823 heartbeater.cc:345] Connected to a master server at 127.0.96.124:38133
I20261120 03:00:42.632027   823 heartbeater.cc:462] Registering TS with master...
I20261120 03:00:42.632454   824 heartbeater.cc:345] Connected to a master server at 127.0.96.126:44467
I20261120 03:00:42.632752   824 heartbeater.cc:462] Registering TS with master...
I20261120 03:00:42.632900   823 heartbeater.cc:508] Master 127.0.96.124:38133 requested a full tablet report, sending...
I20261120 03:00:42.633579   824 heartbeater.cc:508] Master 127.0.96.126:44467 requested a full tablet report, sending...
I20261120 03:00:42.635650   580 ts_manager.cc:210] Registered new tserver with Master: 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:42.636572   502 ts_manager.cc:210] Registered new tserver with Master: 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:42.638788   580 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.0.96.65:43299
I20261120 03:00:42.642443   426 ts_manager.cc:210] Registered new tserver with Master: 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:42.690301   833 raft_consensus.cc:534] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 2599fd9eef7d4ce990c40e2356a37c3f)
I20261120 03:00:42.690703   833 raft_consensus.cc:556] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:42.692849   833 leader_election.cc:290] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e102e07459954b4ea13b65ab9fd5d18b (127.0.96.126:44467), 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133)
I20261120 03:00:42.694126   590 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "18331bef5c884dcab9dbd26bc2c82273" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" is_pre_election: true
I20261120 03:00:42.694092   435 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "18331bef5c884dcab9dbd26bc2c82273" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "e102e07459954b4ea13b65ab9fd5d18b" is_pre_election: true
I20261120 03:00:42.695803   486 leader_election.cc:304] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 18331bef5c884dcab9dbd26bc2c82273; no voters: 2599fd9eef7d4ce990c40e2356a37c3f, e102e07459954b4ea13b65ab9fd5d18b
I20261120 03:00:42.696360   833 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20261120 03:00:43.049738   830 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
W20261120 03:00:43.050297   830 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20261120 03:00:43.050869   830 flags.cc:432] Enabled unsafe flag: --never_fsync=true
W20261120 03:00:43.085409   830 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
W20261120 03:00:43.086457   830 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.96.66
I20261120 03:00:43.130131   830 tablet_server_runner.cc:78] Tablet server non-default flags:
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/logs
--logbuflevel=-1
--logtostderr=true
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.96.84:35145
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/data
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.0.96.66:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/data/info.pb
--webserver_interface=127.0.96.66
--webserver_port=0
--heartbeat_interval_ms=500
--heartbeat_max_failures_before_backoff=1
--heartbeat_rpc_timeout_ms=1000
--tserver_master_addrs=127.0.96.126:44467,127.0.96.125:37411,127.0.96.124:38133
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.0.96.66

Tablet server version:
kudu 1.19.0-SNAPSHOT
revision 36b6772d828145f8fbba3c49746c86838683f9ba
build type FASTDEBUG
built by None at 20 Nov 2026 02:45:53 UTC on bdcb31816ec0
build id 14870
TSAN enabled
I20261120 03:00:43.131673   830 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20261120 03:00:43.133322   830 file_cache.cc:504] Constructed file cache file cache with capacity 419430
I20261120 03:00:43.145372   823 heartbeater.cc:500] Master 127.0.96.124:38133 was elected leader, sending a full tablet report...
W20261120 03:00:43.149012   842 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:00:43.251070   847 raft_consensus.cc:534] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 2599fd9eef7d4ce990c40e2356a37c3f)
I20261120 03:00:43.252915   847 raft_consensus.cc:556] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:43.257603   435 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "18331bef5c884dcab9dbd26bc2c82273" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "e102e07459954b4ea13b65ab9fd5d18b" is_pre_election: true
I20261120 03:00:43.263266   590 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "18331bef5c884dcab9dbd26bc2c82273" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" is_pre_election: true
I20261120 03:00:43.265228   486 leader_election.cc:304] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 18331bef5c884dcab9dbd26bc2c82273; no voters: 2599fd9eef7d4ce990c40e2356a37c3f, e102e07459954b4ea13b65ab9fd5d18b
I20261120 03:00:43.271577   847 leader_election.cc:290] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e102e07459954b4ea13b65ab9fd5d18b (127.0.96.126:44467), 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133)
I20261120 03:00:43.271591   848 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20261120 03:00:44.595906   849 raft_consensus.cc:534] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 2599fd9eef7d4ce990c40e2356a37c3f)
I20261120 03:00:44.596622   849 raft_consensus.cc:556] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:44.600515   590 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e102e07459954b4ea13b65ab9fd5d18b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" is_pre_election: true
I20261120 03:00:44.600759   512 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e102e07459954b4ea13b65ab9fd5d18b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "18331bef5c884dcab9dbd26bc2c82273" is_pre_election: true
I20261120 03:00:44.602358   849 leader_election.cc:290] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 18331bef5c884dcab9dbd26bc2c82273 (127.0.96.125:37411), 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133)
I20261120 03:00:44.602429   412 leader_election.cc:304] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e102e07459954b4ea13b65ab9fd5d18b; no voters: 18331bef5c884dcab9dbd26bc2c82273, 2599fd9eef7d4ce990c40e2356a37c3f
I20261120 03:00:44.603220   849 raft_consensus.cc:2966] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20261120 03:00:44.638183   850 raft_consensus.cc:534] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 2599fd9eef7d4ce990c40e2356a37c3f)
I20261120 03:00:44.638681   850 raft_consensus.cc:556] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:44.640570   850 leader_election.cc:290] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e102e07459954b4ea13b65ab9fd5d18b (127.0.96.126:44467), 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133)
I20261120 03:00:44.642341   435 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "18331bef5c884dcab9dbd26bc2c82273" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "e102e07459954b4ea13b65ab9fd5d18b" is_pre_election: true
I20261120 03:00:44.643061   435 raft_consensus.cc:2062] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 1 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 18331bef5c884dcab9dbd26bc2c82273 in term 1
I20261120 03:00:44.644500   590 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "18331bef5c884dcab9dbd26bc2c82273" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" is_pre_election: true
I20261120 03:00:44.646754   486 leader_election.cc:304] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 18331bef5c884dcab9dbd26bc2c82273, e102e07459954b4ea13b65ab9fd5d18b; no voters: 
I20261120 03:00:44.648176   850 raft_consensus.cc:3024] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Leader pre-election won for term 2
I20261120 03:00:44.648545   850 raft_consensus.cc:534] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 2599fd9eef7d4ce990c40e2356a37c3f)
I20261120 03:00:44.648939   850 raft_consensus.cc:3296] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 1 FOLLOWER]: Advancing to term 2
I20261120 03:00:44.678398   850 raft_consensus.cc:556] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:44.688688   850 leader_election.cc:290] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [CANDIDATE]: Term 2 election: Requested vote from peers e102e07459954b4ea13b65ab9fd5d18b (127.0.96.126:44467), 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133)
I20261120 03:00:44.692420   590 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "18331bef5c884dcab9dbd26bc2c82273" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2599fd9eef7d4ce990c40e2356a37c3f"
I20261120 03:00:44.690373   435 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "18331bef5c884dcab9dbd26bc2c82273" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "e102e07459954b4ea13b65ab9fd5d18b"
I20261120 03:00:44.697860   486 leader_election.cc:304] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 18331bef5c884dcab9dbd26bc2c82273; no voters: 2599fd9eef7d4ce990c40e2356a37c3f, e102e07459954b4ea13b65ab9fd5d18b
I20261120 03:00:44.699074   851 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
W20261120 03:00:44.548964   840 debug-util.cc:398] Leaking SignalData structure 0x7b0800032a00 after lost signal to thread 830
W20261120 03:00:43.151364   841 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:00:44.775242   830 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.625s	user 0.650s	sys 0.926s
W20261120 03:00:44.776134   830 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.626s	user 0.650s	sys 0.926s
W20261120 03:00:44.777300   844 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:00:44.780682   843 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1626 milliseconds
I20261120 03:00:44.780735   830 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20261120 03:00:44.781852   830 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20261120 03:00:44.783965   830 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20261120 03:00:44.785315   830 hybrid_clock.cc:648] HybridClock initialized: now 1795143644785287 us; error 32 us; skew 500 ppm
I20261120 03:00:44.786114   830 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20261120 03:00:44.792392   830 webserver.cc:577] Webserver started at http://127.0.96.66:35063/ using document root <none> and password file <none>
I20261120 03:00:44.793278   830 fs_manager.cc:362] Metadata directory not provided
I20261120 03:00:44.793483   830 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20261120 03:00:44.793929   830 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
I20261120 03:00:44.798034   830 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/data/instance:
uuid: "3d30c2428730481db9b0ffd279dd3440"
format_stamp: "Formatted at 2026-11-20 03:00:44 on dist-test-slave-jpwh"
I20261120 03:00:44.799182   830 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/instance:
uuid: "3d30c2428730481db9b0ffd279dd3440"
format_stamp: "Formatted at 2026-11-20 03:00:44 on dist-test-slave-jpwh"
I20261120 03:00:44.806203   830 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.006s	sys 0.001s
I20261120 03:00:44.811808   856 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:00:44.812866   830 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
I20261120 03:00:44.813189   830 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/data,/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal
uuid: "3d30c2428730481db9b0ffd279dd3440"
format_stamp: "Formatted at 2026-11-20 03:00:44 on dist-test-slave-jpwh"
I20261120 03:00:44.813498   830 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal
metadata directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal
1 data directories: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20261120 03:00:44.860781   830 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20261120 03:00:44.862299   830 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20261120 03:00:44.862722   830 kserver.cc:163] Server-wide thread pool size limit: 3276
I20261120 03:00:44.865710   830 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20261120 03:00:44.869796   830 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20261120 03:00:44.870003   830 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:00:44.870291   830 ts_tablet_manager.cc:662] Registered 0 tablets
I20261120 03:00:44.870462   830 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:00:45.022864   830 rpc_server.cc:307] RPC server started. Bound to: 127.0.96.66:42987
I20261120 03:00:45.022969   968 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.96.66:42987 every 8 connection(s)
I20261120 03:00:45.025580   830 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/data/info.pb
I20261120 03:00:45.028044   385 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu as pid 830
I20261120 03:00:45.028739   385 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/instance
I20261120 03:00:45.053027   385 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
/tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/logs
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.96.67:0
--local_ip_for_outbound_sockets=127.0.96.67
--webserver_interface=127.0.96.67
--webserver_port=0
--tserver_master_addrs=127.0.96.126:44467,127.0.96.125:37411,127.0.96.124:38133
--builtin_ntp_servers=127.0.96.84:35145
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--heartbeat_rpc_timeout_ms=1000
--heartbeat_max_failures_before_backoff=1
--heartbeat_interval_ms=500 with env {}
I20261120 03:00:45.083498   969 heartbeater.cc:345] Connected to a master server at 127.0.96.125:37411
I20261120 03:00:45.084003   969 heartbeater.cc:462] Registering TS with master...
I20261120 03:00:45.085498   969 heartbeater.cc:508] Master 127.0.96.125:37411 requested a full tablet report, sending...
I20261120 03:00:45.089401   502 ts_manager.cc:210] Registered new tserver with Master: 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:45.101465   970 heartbeater.cc:345] Connected to a master server at 127.0.96.124:38133
I20261120 03:00:45.101840   970 heartbeater.cc:462] Registering TS with master...
I20261120 03:00:45.102703   970 heartbeater.cc:508] Master 127.0.96.124:38133 requested a full tablet report, sending...
I20261120 03:00:45.105512   580 ts_manager.cc:210] Registered new tserver with Master: 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:45.107133   580 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.0.96.66:33595
I20261120 03:00:45.107882   971 heartbeater.cc:345] Connected to a master server at 127.0.96.126:44467
I20261120 03:00:45.108209   971 heartbeater.cc:462] Registering TS with master...
I20261120 03:00:45.109390   971 heartbeater.cc:508] Master 127.0.96.126:44467 requested a full tablet report, sending...
I20261120 03:00:45.111800   426 ts_manager.cc:210] Registered new tserver with Master: 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:45.186899   512 raft_consensus.cc:1303] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 2 FOLLOWER]: Rejecting Update request from peer 2599fd9eef7d4ce990c40e2356a37c3f for earlier term 1. Current term is 2. Ops: []
I20261120 03:00:45.188341   647 consensus_queue.cc:1060] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [LEADER]: Peer responded invalid term: Peer: permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 }, Status: INVALID_TERM, Last received: 1.4, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20261120 03:00:45.189759   980 raft_consensus.cc:3291] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 1 LEADER]: Stepping down as leader of term 1
I20261120 03:00:45.190091   980 raft_consensus.cc:787] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 2599fd9eef7d4ce990c40e2356a37c3f, State: Running, Role: LEADER
I20261120 03:00:45.190790   980 consensus_queue.cc:261] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:45.191668   980 raft_consensus.cc:3296] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 1 FOLLOWER]: Advancing to term 2
I20261120 03:00:45.287228   981 raft_consensus.cc:534] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:45.287690   981 raft_consensus.cc:556] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:45.300108   435 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "18331bef5c884dcab9dbd26bc2c82273" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "e102e07459954b4ea13b65ab9fd5d18b" is_pre_election: true
I20261120 03:00:45.300271   590 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "18331bef5c884dcab9dbd26bc2c82273" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" is_pre_election: true
I20261120 03:00:45.300897   590 raft_consensus.cc:2062] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 2 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 18331bef5c884dcab9dbd26bc2c82273 in term 2
I20261120 03:00:45.289697   981 leader_election.cc:290] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers e102e07459954b4ea13b65ab9fd5d18b (127.0.96.126:44467), 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133)
I20261120 03:00:45.302577   486 leader_election.cc:304] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 18331bef5c884dcab9dbd26bc2c82273, 2599fd9eef7d4ce990c40e2356a37c3f; no voters: e102e07459954b4ea13b65ab9fd5d18b
I20261120 03:00:45.303592   981 raft_consensus.cc:3024] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 2 FOLLOWER]: Leader pre-election won for term 3
I20261120 03:00:45.304005   981 raft_consensus.cc:534] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:45.304289   981 raft_consensus.cc:3296] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 2 FOLLOWER]: Advancing to term 3
I20261120 03:00:45.305616   981 raft_consensus.cc:556] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:45.312964   981 leader_election.cc:290] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [CANDIDATE]: Term 3 election: Requested vote from peers e102e07459954b4ea13b65ab9fd5d18b (127.0.96.126:44467), 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133)
I20261120 03:00:45.313951   590 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "18331bef5c884dcab9dbd26bc2c82273" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2599fd9eef7d4ce990c40e2356a37c3f"
I20261120 03:00:45.314062   435 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "18331bef5c884dcab9dbd26bc2c82273" candidate_term: 3 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "e102e07459954b4ea13b65ab9fd5d18b"
I20261120 03:00:45.314421   590 raft_consensus.cc:3296] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 2 FOLLOWER]: Advancing to term 3
I20261120 03:00:45.315230   590 raft_consensus.cc:2062] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 3 FOLLOWER]: Leader election vote request: granting yes vote for candidate 18331bef5c884dcab9dbd26bc2c82273 in term 3
I20261120 03:00:45.320741   486 leader_election.cc:304] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 18331bef5c884dcab9dbd26bc2c82273, 2599fd9eef7d4ce990c40e2356a37c3f; no voters: e102e07459954b4ea13b65ab9fd5d18b
I20261120 03:00:45.321579   981 raft_consensus.cc:3024] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 3 FOLLOWER]: Leader election won for term 3
I20261120 03:00:45.322252   981 raft_consensus.cc:744] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [term 3 LEADER]: Becoming Leader. State: Replica: 18331bef5c884dcab9dbd26bc2c82273, State: Running, Role: LEADER
I20261120 03:00:45.323139   981 consensus_queue.cc:238] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:45.327739   983 sys_catalog.cc:455] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 18331bef5c884dcab9dbd26bc2c82273. Latest consensus state: current_term: 3 leader_uuid: "18331bef5c884dcab9dbd26bc2c82273" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:45.328649   983 sys_catalog.cc:458] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [sys.catalog]: This master's current role is: LEADER
I20261120 03:00:45.331378   986 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
I20261120 03:00:45.334256   435 raft_consensus.cc:3296] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 1 FOLLOWER]: Advancing to term 3
I20261120 03:00:45.334312   590 raft_consensus.cc:1338] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 3 FOLLOWER]: Refusing update from remote peer 18331bef5c884dcab9dbd26bc2c82273: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 5. (index mismatch)
I20261120 03:00:45.335879   435 raft_consensus.cc:1338] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 3 FOLLOWER]: Refusing update from remote peer 18331bef5c884dcab9dbd26bc2c82273: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 5. (index mismatch)
I20261120 03:00:45.336411   985 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
I20261120 03:00:45.345124   986 catalog_manager.cc:1535] Initializing Kudu cluster ID...
I20261120 03:00:45.349344   986 catalog_manager.cc:1310] Loaded cluster ID: 96a176325a9f4b23b7e54922a0ed8e6d
I20261120 03:00:45.349687   986 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
I20261120 03:00:45.349527   985 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20261120 03:00:45.351945   980 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 18331bef5c884dcab9dbd26bc2c82273. Latest consensus state: current_term: 3 leader_uuid: "18331bef5c884dcab9dbd26bc2c82273" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:45.352639   980 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:00:45.354816   986 catalog_manager.cc:1555] Loading token signing keys...
I20261120 03:00:45.359259   986 catalog_manager.cc:6248] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Loaded TSK: 0
I20261120 03:00:45.360760   986 catalog_manager.cc:1565] Initializing in-progress tserver states...
I20261120 03:00:45.362255   987 sys_catalog.cc:455] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "18331bef5c884dcab9dbd26bc2c82273" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:45.363175   987 sys_catalog.cc:458] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [sys.catalog]: This master's current role is: LEADER
I20261120 03:00:45.363704   988 sys_catalog.cc:455] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 18331bef5c884dcab9dbd26bc2c82273. Latest consensus state: current_term: 3 leader_uuid: "18331bef5c884dcab9dbd26bc2c82273" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:45.364702   988 sys_catalog.cc:458] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:00:45.366860   980 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "18331bef5c884dcab9dbd26bc2c82273" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:45.367549   980 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:00:45.367924   981 sys_catalog.cc:455] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "18331bef5c884dcab9dbd26bc2c82273" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:45.368499   981 sys_catalog.cc:458] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [sys.catalog]: This master's current role is: LEADER
I20261120 03:00:45.371367   988 sys_catalog.cc:455] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "18331bef5c884dcab9dbd26bc2c82273" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:45.371976   988 sys_catalog.cc:458] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [sys.catalog]: This master's current role is: FOLLOWER
W20261120 03:00:45.421386   977 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
W20261120 03:00:45.421900   977 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20261120 03:00:45.422523   977 flags.cc:432] Enabled unsafe flag: --never_fsync=true
W20261120 03:00:45.455876   977 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
W20261120 03:00:45.456863   977 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.96.67
I20261120 03:00:45.494930   977 tablet_server_runner.cc:78] Tablet server non-default flags:
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/logs
--logbuflevel=-1
--logtostderr=true
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.96.84:35145
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/data
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.0.96.67:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/data/info.pb
--webserver_interface=127.0.96.67
--webserver_port=0
--heartbeat_interval_ms=500
--heartbeat_max_failures_before_backoff=1
--heartbeat_rpc_timeout_ms=1000
--tserver_master_addrs=127.0.96.126:44467,127.0.96.125:37411,127.0.96.124:38133
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.0.96.67

Tablet server version:
kudu 1.19.0-SNAPSHOT
revision 36b6772d828145f8fbba3c49746c86838683f9ba
build type FASTDEBUG
built by None at 20 Nov 2026 02:45:53 UTC on bdcb31816ec0
build id 14870
TSAN enabled
I20261120 03:00:45.496291   977 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20261120 03:00:45.497947   977 file_cache.cc:504] Constructed file cache file cache with capacity 419430
W20261120 03:00:45.511065   999 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:00:45.610495   970 heartbeater.cc:500] Master 127.0.96.124:38133 was elected leader, sending a full tablet report...
I20261120 03:00:45.780788   639 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
I20261120 03:00:45.857966   980 raft_consensus.cc:534] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 18331bef5c884dcab9dbd26bc2c82273)
I20261120 03:00:45.858413   980 raft_consensus.cc:556] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:45.860018   980 leader_election.cc:290] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers e102e07459954b4ea13b65ab9fd5d18b (127.0.96.126:44467), 18331bef5c884dcab9dbd26bc2c82273 (127.0.96.125:37411)
I20261120 03:00:45.860991   435 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "e102e07459954b4ea13b65ab9fd5d18b" is_pre_election: true
I20261120 03:00:45.861358   512 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "18331bef5c884dcab9dbd26bc2c82273" is_pre_election: true
I20261120 03:00:45.863030   565 leader_election.cc:304] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2599fd9eef7d4ce990c40e2356a37c3f; no voters: 18331bef5c884dcab9dbd26bc2c82273, e102e07459954b4ea13b65ab9fd5d18b
I20261120 03:00:45.863669   980 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20261120 03:00:46.096568   969 heartbeater.cc:500] Master 127.0.96.125:37411 was elected leader, sending a full tablet report...
I20261120 03:00:46.189064   822 heartbeater.cc:500] Master 127.0.96.125:37411 was elected leader, sending a full tablet report...
I20261120 03:00:46.389890  1005 raft_consensus.cc:534] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 18331bef5c884dcab9dbd26bc2c82273)
I20261120 03:00:46.390322  1005 raft_consensus.cc:556] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:46.391846  1005 leader_election.cc:290] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers e102e07459954b4ea13b65ab9fd5d18b (127.0.96.126:44467), 18331bef5c884dcab9dbd26bc2c82273 (127.0.96.125:37411)
I20261120 03:00:46.392913   512 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "18331bef5c884dcab9dbd26bc2c82273" is_pre_election: true
I20261120 03:00:46.393050   435 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "e102e07459954b4ea13b65ab9fd5d18b" is_pre_election: true
I20261120 03:00:46.394390   564 leader_election.cc:304] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2599fd9eef7d4ce990c40e2356a37c3f; no voters: 18331bef5c884dcab9dbd26bc2c82273, e102e07459954b4ea13b65ab9fd5d18b
I20261120 03:00:46.394974  1005 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
W20261120 03:00:45.511386  1000 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:00:45.515025  1002 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:00:46.611002  1001 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
I20261120 03:00:46.611043   977 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20261120 03:00:46.615157   977 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20261120 03:00:46.617367   977 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20261120 03:00:46.618825   977 hybrid_clock.cc:648] HybridClock initialized: now 1795143646618774 us; error 49 us; skew 500 ppm
I20261120 03:00:46.619858   977 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20261120 03:00:46.627712   977 webserver.cc:577] Webserver started at http://127.0.96.67:40165/ using document root <none> and password file <none>
I20261120 03:00:46.628970   977 fs_manager.cc:362] Metadata directory not provided
I20261120 03:00:46.629170   977 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20261120 03:00:46.629585   977 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
I20261120 03:00:46.633854   977 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/data/instance:
uuid: "823abdbd90ba47bbacad0f8dcff78234"
format_stamp: "Formatted at 2026-11-20 03:00:46 on dist-test-slave-jpwh"
I20261120 03:00:46.635146   977 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/instance:
uuid: "823abdbd90ba47bbacad0f8dcff78234"
format_stamp: "Formatted at 2026-11-20 03:00:46 on dist-test-slave-jpwh"
I20261120 03:00:46.643244   977 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.007s	sys 0.001s
I20261120 03:00:46.649170  1010 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:00:46.650357   977 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.000s
I20261120 03:00:46.650663   977 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/data,/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal
uuid: "823abdbd90ba47bbacad0f8dcff78234"
format_stamp: "Formatted at 2026-11-20 03:00:46 on dist-test-slave-jpwh"
I20261120 03:00:46.651003   977 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal
metadata directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal
1 data directories: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20261120 03:00:46.697465   977 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20261120 03:00:46.698971   977 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20261120 03:00:46.699404   977 kserver.cc:163] Server-wide thread pool size limit: 3276
I20261120 03:00:46.702387   977 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20261120 03:00:46.706640   977 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20261120 03:00:46.706869   977 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:00:46.707149   977 ts_tablet_manager.cc:662] Registered 0 tablets
I20261120 03:00:46.707362   977 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:00:46.728742  1027 raft_consensus.cc:534] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 18331bef5c884dcab9dbd26bc2c82273)
I20261120 03:00:46.729290  1027 raft_consensus.cc:556] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:46.731457  1027 leader_election.cc:290] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 18331bef5c884dcab9dbd26bc2c82273 (127.0.96.125:37411), 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133)
I20261120 03:00:46.732649   512 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e102e07459954b4ea13b65ab9fd5d18b" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "18331bef5c884dcab9dbd26bc2c82273" is_pre_election: true
I20261120 03:00:46.732640   589 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e102e07459954b4ea13b65ab9fd5d18b" candidate_term: 4 candidate_status { last_received { term: 3 index: 5 } } ignore_live_leader: false dest_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" is_pre_election: true
I20261120 03:00:46.734604   412 leader_election.cc:304] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e102e07459954b4ea13b65ab9fd5d18b; no voters: 18331bef5c884dcab9dbd26bc2c82273, 2599fd9eef7d4ce990c40e2356a37c3f
I20261120 03:00:46.735409  1027 raft_consensus.cc:2966] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20261120 03:00:46.885820   977 rpc_server.cc:307] RPC server started. Bound to: 127.0.96.67:34451
I20261120 03:00:46.885883  1123 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.96.67:34451 every 8 connection(s)
I20261120 03:00:46.889443   977 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/data/info.pb
I20261120 03:00:46.889890   385 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu as pid 977
I20261120 03:00:46.890288   385 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/instance
I20261120 03:00:46.940809  1124 heartbeater.cc:345] Connected to a master server at 127.0.96.125:37411
I20261120 03:00:46.941342  1124 heartbeater.cc:462] Registering TS with master...
I20261120 03:00:46.950388  1124 heartbeater.cc:508] Master 127.0.96.125:37411 requested a full tablet report, sending...
I20261120 03:00:46.953434   502 ts_manager.cc:210] Registered new tserver with Master: 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:46.956162   502 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.0.96.67:53803
I20261120 03:00:46.960770  1125 heartbeater.cc:345] Connected to a master server at 127.0.96.124:38133
I20261120 03:00:46.961122  1125 heartbeater.cc:462] Registering TS with master...
I20261120 03:00:46.961764  1125 heartbeater.cc:508] Master 127.0.96.124:38133 requested a full tablet report, sending...
I20261120 03:00:46.963665  1128 heartbeater.cc:345] Connected to a master server at 127.0.96.126:44467
I20261120 03:00:46.963943  1128 heartbeater.cc:462] Registering TS with master...
I20261120 03:00:46.964241   580 ts_manager.cc:210] Registered new tserver with Master: 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:46.964540  1128 heartbeater.cc:508] Master 127.0.96.126:44467 requested a full tablet report, sending...
I20261120 03:00:46.966801   426 ts_manager.cc:210] Registered new tserver with Master: 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:46.976796   385 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
I20261120 03:00:47.032536   502 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_60efbe3d83f545c4944277942831bdb1"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:47.033742   501 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_f1aba2854d064ec480c1c86e806f03b4"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:47.035734   502 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_60efbe3d83f545c4944277942831bdb1 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:00:47.035840   501 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_f1aba2854d064ec480c1c86e806f03b4 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:47.037456   500 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_c3a6b4f081eb4eee8d73165b8394d378"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:47.038470   499 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_8c5570a27f5245288d0ae25060cc31eb"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:47.040414   500 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_c3a6b4f081eb4eee8d73165b8394d378 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:00:47.041170   499 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_8c5570a27f5245288d0ae25060cc31eb in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:47.152273   500 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_c3a6b4f081eb4eee8d73165b8394d378" } modify_external_catalogs: true
I20261120 03:00:47.164181   500 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_c3a6b4f081eb4eee8d73165b8394d378" } modify_external_catalogs: true
I20261120 03:00:47.155606   502 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_60efbe3d83f545c4944277942831bdb1" } new_table_name: "default.Table_83bc25a7e46247daae268a2d2db4ac10" modify_external_catalogs: true
I20261120 03:00:47.166638   499 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_8c5570a27f5245288d0ae25060cc31eb" } modify_external_catalogs: true
I20261120 03:00:47.167187   499 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_8c5570a27f5245288d0ae25060cc31eb" } modify_external_catalogs: true
I20261120 03:00:47.157490   501 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_f1aba2854d064ec480c1c86e806f03b4" } new_table_name: "default.Table_749e6064fff14577a78fa67002d19c78" modify_external_catalogs: true
I20261120 03:00:47.184969   385 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu with pid 539
I20261120 03:00:47.203205   499 catalog_manager.cc:6146] Not sending DeleteTablet requests; no consensus state for tablet 0b98dd3d5ecd4955b7494fd83445d433
I20261120 03:00:47.214061   500 catalog_manager.cc:6146] Not sending DeleteTablet requests; no consensus state for tablet 8b74f0481f364b16a3e0a610bb0db25a
W20261120 03:00:47.238574   709 connection.cc:573] client connection to 127.0.96.124:38133 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:00:47.239451   823 heartbeater.cc:648] Failed to heartbeat to 127.0.96.124:38133 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:00:47.239821   486 connection.cc:573] client connection to 127.0.96.124:38133 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:00:47.240217   486 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Connection reset by peer (error 104) [suppressed 6 similar messages]
W20261120 03:00:47.258879   501 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 1cf6d449fa2845a7b1f4cbd1de639d20 has no consensus state
W20261120 03:00:47.261359   502 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 742fab63984e4752ac6c62d32ae39ae7 consensus state has no leader
W20261120 03:00:47.266235   486 consensus_peers.cc:597] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 -> Peer 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133): Couldn't send request to peer 2599fd9eef7d4ce990c40e2356a37c3f. Status: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:47.276661   904 tablet_service.cc:1495] Processing CreateTablet for tablet 742fab63984e4752ac6c62d32ae39ae7 (DEFAULT_TABLE table=default.Table_60efbe3d83f545c4944277942831bdb1 [id=bb08bbcb4357420398926675042c3cce]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.278599   904 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 742fab63984e4752ac6c62d32ae39ae7. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.282447   903 tablet_service.cc:1495] Processing CreateTablet for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (DEFAULT_TABLE table=default.Table_749e6064fff14577a78fa67002d19c78 [id=cfcd5fea97fb4ab3baff99eb28cd9476]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.280258  1059 tablet_service.cc:1495] Processing CreateTablet for tablet 742fab63984e4752ac6c62d32ae39ae7 (DEFAULT_TABLE table=default.Table_60efbe3d83f545c4944277942831bdb1 [id=bb08bbcb4357420398926675042c3cce]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.284698   756 tablet_service.cc:1495] Processing CreateTablet for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (DEFAULT_TABLE table=default.Table_749e6064fff14577a78fa67002d19c78 [id=cfcd5fea97fb4ab3baff99eb28cd9476]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.273599   757 tablet_service.cc:1495] Processing CreateTablet for tablet 742fab63984e4752ac6c62d32ae39ae7 (DEFAULT_TABLE table=default.Table_60efbe3d83f545c4944277942831bdb1 [id=bb08bbcb4357420398926675042c3cce]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.285599  1058 tablet_service.cc:1495] Processing CreateTablet for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (DEFAULT_TABLE table=default.Table_749e6064fff14577a78fa67002d19c78 [id=cfcd5fea97fb4ab3baff99eb28cd9476]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.286813   756 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1cf6d449fa2845a7b1f4cbd1de639d20. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.287587   757 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 742fab63984e4752ac6c62d32ae39ae7. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.288017  1058 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1cf6d449fa2845a7b1f4cbd1de639d20. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.289799   903 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1cf6d449fa2845a7b1f4cbd1de639d20. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.298784  1059 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 742fab63984e4752ac6c62d32ae39ae7. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:00:47.306658   486 catalog_manager.cc:4863] Async tablet task AlterTable RPC for tablet 742fab63984e4752ac6c62d32ae39ae7 (table default.Table_83bc25a7e46247daae268a2d2db4ac10 [id=bb08bbcb4357420398926675042c3cce], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 742fab63984e4752ac6c62d32ae39ae7 consensus state has no leader
I20261120 03:00:47.328943   502 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_828236dc53434d038b86eb38ebacf7c6"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:47.331537   502 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_828236dc53434d038b86eb38ebacf7c6 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:47.340553   501 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_9b27b75009eb476c9d595545e2c25b55"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:47.342263   501 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_9b27b75009eb476c9d595545e2c25b55 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:47.343210   500 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_43bea6dc28bd4b8ca2b09c031d656936"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:47.344875   500 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_43bea6dc28bd4b8ca2b09c031d656936 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:47.352406  1180 tablet_bootstrap.cc:492] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:47.361187  1180 tablet_bootstrap.cc:654] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:47.367017  1180 log.cc:826] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440: Log is configured to *not* fsync() on all Append() calls
I20261120 03:00:47.367602   501 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_c2495e8bbaf649548720a83aefc281f9"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:47.369293   501 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_c2495e8bbaf649548720a83aefc281f9 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:47.379840  1181 tablet_bootstrap.cc:492] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:47.386142   903 tablet_service.cc:1495] Processing CreateTablet for tablet ec08ada9d82b4692a496fef3f8c53807 (DEFAULT_TABLE table=default.Table_828236dc53434d038b86eb38ebacf7c6 [id=c4ac173f0a0e41df96cc9da60adc2ab1]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.387315   903 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ec08ada9d82b4692a496fef3f8c53807. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.387971   755 tablet_service.cc:1495] Processing CreateTablet for tablet ec08ada9d82b4692a496fef3f8c53807 (DEFAULT_TABLE table=default.Table_828236dc53434d038b86eb38ebacf7c6 [id=c4ac173f0a0e41df96cc9da60adc2ab1]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.388888  1181 tablet_bootstrap.cc:654] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:47.389209   755 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ec08ada9d82b4692a496fef3f8c53807. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.391237  1181 log.cc:826] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234: Log is configured to *not* fsync() on all Append() calls
I20261120 03:00:47.391842  1059 tablet_service.cc:1495] Processing CreateTablet for tablet ec08ada9d82b4692a496fef3f8c53807 (DEFAULT_TABLE table=default.Table_828236dc53434d038b86eb38ebacf7c6 [id=c4ac173f0a0e41df96cc9da60adc2ab1]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.392860  1059 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ec08ada9d82b4692a496fef3f8c53807. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.406172  1181 tablet_bootstrap.cc:492] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:47.408946   502 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_9b27b75009eb476c9d595545e2c25b55" } modify_external_catalogs: true
I20261120 03:00:47.409595   502 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_9b27b75009eb476c9d595545e2c25b55" } modify_external_catalogs: true
I20261120 03:00:47.414620  1181 ts_tablet_manager.cc:1449] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.064s	user 0.010s	sys 0.012s
I20261120 03:00:47.435251   754 tablet_service.cc:1495] Processing CreateTablet for tablet 27cfe2018ee44ab7beaa59992dc6250a (DEFAULT_TABLE table=default.Table_9b27b75009eb476c9d595545e2c25b55 [id=3937e02f9ba6411fbf5e91e2cdb235d3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.436900   903 tablet_service.cc:1495] Processing CreateTablet for tablet 27cfe2018ee44ab7beaa59992dc6250a (DEFAULT_TABLE table=default.Table_9b27b75009eb476c9d595545e2c25b55 [id=3937e02f9ba6411fbf5e91e2cdb235d3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.438208   903 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 27cfe2018ee44ab7beaa59992dc6250a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.444540   502 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 27cfe2018ee44ab7beaa59992dc6250a
I20261120 03:00:47.445891  1058 tablet_service.cc:1495] Processing CreateTablet for tablet 27cfe2018ee44ab7beaa59992dc6250a (DEFAULT_TABLE table=default.Table_9b27b75009eb476c9d595545e2c25b55 [id=3937e02f9ba6411fbf5e91e2cdb235d3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.447242  1058 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 27cfe2018ee44ab7beaa59992dc6250a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.450486   502 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_c2495e8bbaf649548720a83aefc281f9" } modify_external_catalogs: true
I20261120 03:00:47.450819   904 tablet_service.cc:1542] Processing DeleteTablet for tablet 27cfe2018ee44ab7beaa59992dc6250a with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:47.451020   502 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_c2495e8bbaf649548720a83aefc281f9" } modify_external_catalogs: true
I20261120 03:00:47.453987   754 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 27cfe2018ee44ab7beaa59992dc6250a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.453882   753 tablet_service.cc:1542] Processing DeleteTablet for tablet 27cfe2018ee44ab7beaa59992dc6250a with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:47.455560  1057 tablet_service.cc:1542] Processing DeleteTablet for tablet 27cfe2018ee44ab7beaa59992dc6250a with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:47.458120  1181 raft_consensus.cc:366] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.459088  1181 raft_consensus.cc:392] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:47.459394  1181 raft_consensus.cc:787] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:47.460076  1124 heartbeater.cc:500] Master 127.0.96.125:37411 was elected leader, sending a full tablet report...
I20261120 03:00:47.461004  1181 consensus_queue.cc:261] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.467248   501 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_43bea6dc28bd4b8ca2b09c031d656936" } new_table_name: "default.Table_5e08feb4f2304718b4cc6c71a638e14c" modify_external_catalogs: true
I20261120 03:00:47.467352   500 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_83bc25a7e46247daae268a2d2db4ac10" } new_table_name: "default.Table_c0ee25431db34324af521703039717fe" modify_external_catalogs: true
I20261120 03:00:47.500092   502 catalog_manager.cc:6146] Not sending DeleteTablet requests; no consensus state for tablet d2aad7a50dc84724a774a05c46a95d34
I20261120 03:00:47.504663  1189 tablet_bootstrap.cc:492] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:47.507933  1190 raft_consensus.cc:534] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:47.508496  1190 raft_consensus.cc:556] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
W20261120 03:00:47.511256   501 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 5328cf9c683a44548038a1b585b84e94 has no consensus state
W20261120 03:00:47.532933   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 27cfe2018ee44ab7beaa59992dc6250a because the tablet was not found. No further retry: Not found: Tablet not found: 27cfe2018ee44ab7beaa59992dc6250a
I20261120 03:00:47.538056   902 tablet_service.cc:1495] Processing CreateTablet for tablet 5328cf9c683a44548038a1b585b84e94 (DEFAULT_TABLE table=default.Table_5e08feb4f2304718b4cc6c71a638e14c [id=59c24564b2eb44ccab8736e0ef5f733b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.539448   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5328cf9c683a44548038a1b585b84e94. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.541426   499 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 742fab63984e4752ac6c62d32ae39ae7 (table default.Table_83bc25a7e46247daae268a2d2db4ac10 [id=bb08bbcb4357420398926675042c3cce]). Expected version 1 got 0
I20261120 03:00:47.542162   499 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (table default.Table_749e6064fff14577a78fa67002d19c78 [id=cfcd5fea97fb4ab3baff99eb28cd9476]). Expected version 1 got 0
W20261120 03:00:47.543076   500 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 742fab63984e4752ac6c62d32ae39ae7 consensus state has no leader
I20261120 03:00:47.545202  1181 ts_tablet_manager.cc:1480] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.130s	user 0.034s	sys 0.011s
I20261120 03:00:47.546260  1181 tablet_bootstrap.cc:492] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:47.548830   757 tablet_service.cc:1495] Processing CreateTablet for tablet 5328cf9c683a44548038a1b585b84e94 (DEFAULT_TABLE table=default.Table_5e08feb4f2304718b4cc6c71a638e14c [id=59c24564b2eb44ccab8736e0ef5f733b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.549168  1056 tablet_service.cc:1495] Processing CreateTablet for tablet 5328cf9c683a44548038a1b585b84e94 (DEFAULT_TABLE table=default.Table_5e08feb4f2304718b4cc6c71a638e14c [id=59c24564b2eb44ccab8736e0ef5f733b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.550122   757 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5328cf9c683a44548038a1b585b84e94. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.550277  1056 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5328cf9c683a44548038a1b585b84e94. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:00:47.553248   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 27cfe2018ee44ab7beaa59992dc6250a because the tablet was not found. No further retry: Not found: Tablet not found: 27cfe2018ee44ab7beaa59992dc6250a
W20261120 03:00:47.557108   486 catalog_manager.cc:5061] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 27cfe2018ee44ab7beaa59992dc6250a because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 27cfe2018ee44ab7beaa59992dc6250a already in progress: creating tablet
I20261120 03:00:47.561518  1189 tablet_bootstrap.cc:654] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:47.565495   499 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_091012122fb94f08a8a199b60c0b2f54"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:47.567682   499 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_091012122fb94f08a8a199b60c0b2f54 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:47.569280   500 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_2171e34de4d74c638b0adec7f3265108"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:47.571030   500 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_2171e34de4d74c638b0adec7f3265108 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:47.579985   501 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 742fab63984e4752ac6c62d32ae39ae7 (table default.Table_c0ee25431db34324af521703039717fe [id=bb08bbcb4357420398926675042c3cce]). Expected version 2 got 0
I20261120 03:00:47.582285  1189 log.cc:826] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6: Log is configured to *not* fsync() on all Append() calls
I20261120 03:00:47.583254  1190 leader_election.cc:290] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:47.585444  1180 tablet_bootstrap.cc:492] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
W20261120 03:00:47.585516  1125 heartbeater.cc:648] Failed to heartbeat to 127.0.96.124:38133 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
I20261120 03:00:47.585997  1180 ts_tablet_manager.cc:1449] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.234s	user 0.035s	sys 0.016s
I20261120 03:00:47.614948   500 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_d57abf08233c42ae9477476dba610f1e"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:47.616866   500 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d57abf08233c42ae9477476dba610f1e in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:47.621462   499 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_185133dc409b4978acdea3f7fd4c0774"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:47.623430   499 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_185133dc409b4978acdea3f7fd4c0774 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:47.642531   757 tablet_service.cc:1495] Processing CreateTablet for tablet a48ff491d14e40b3ba50bf5bde1757f0 (DEFAULT_TABLE table=default.Table_091012122fb94f08a8a199b60c0b2f54 [id=17ac079fd9504c4285a664a9079f1205]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.643648   757 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a48ff491d14e40b3ba50bf5bde1757f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.645208   904 tablet_service.cc:1495] Processing CreateTablet for tablet a48ff491d14e40b3ba50bf5bde1757f0 (DEFAULT_TABLE table=default.Table_091012122fb94f08a8a199b60c0b2f54 [id=17ac079fd9504c4285a664a9079f1205]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.646479   904 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a48ff491d14e40b3ba50bf5bde1757f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.650377   501 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_5e08feb4f2304718b4cc6c71a638e14c" } modify_external_catalogs: true
I20261120 03:00:47.650921   501 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_5e08feb4f2304718b4cc6c71a638e14c" } modify_external_catalogs: true
I20261120 03:00:47.653762  1059 tablet_service.cc:1495] Processing CreateTablet for tablet a48ff491d14e40b3ba50bf5bde1757f0 (DEFAULT_TABLE table=default.Table_091012122fb94f08a8a199b60c0b2f54 [id=17ac079fd9504c4285a664a9079f1205]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.655017  1059 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a48ff491d14e40b3ba50bf5bde1757f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.663334  1179 sys_catalog.cc:455] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "18331bef5c884dcab9dbd26bc2c82273" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:47.668103  1179 sys_catalog.cc:458] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [sys.catalog]: This master's current role is: LEADER
I20261120 03:00:47.670393  1189 tablet_bootstrap.cc:492] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:47.671169  1181 tablet_bootstrap.cc:654] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:47.671260  1189 ts_tablet_manager.cc:1449] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.167s	user 0.032s	sys 0.012s
W20261120 03:00:47.683696  1129 tablet.cc:2554] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20261120 03:00:47.703554   500 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_2171e34de4d74c638b0adec7f3265108" } modify_external_catalogs: true
I20261120 03:00:47.704277   500 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_2171e34de4d74c638b0adec7f3265108" } modify_external_catalogs: true
W20261120 03:00:47.710104   970 heartbeater.cc:648] Failed to heartbeat to 127.0.96.124:38133 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
I20261120 03:00:47.711552   501 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 5328cf9c683a44548038a1b585b84e94
I20261120 03:00:47.714030   757 tablet_service.cc:1542] Processing DeleteTablet for tablet 5328cf9c683a44548038a1b585b84e94 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:47.714669   901 tablet_service.cc:1542] Processing DeleteTablet for tablet 5328cf9c683a44548038a1b585b84e94 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:47.715744   502 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_749e6064fff14577a78fa67002d19c78" } new_table_name: "default.Table_34bfe641479b42ceac031dafd7e21d2d" modify_external_catalogs: true
W20261120 03:00:47.717069   487 catalog_manager.cc:5061] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 5328cf9c683a44548038a1b585b84e94 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 5328cf9c683a44548038a1b585b84e94 already in progress: creating tablet
W20261120 03:00:47.719223   486 catalog_manager.cc:5061] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 5328cf9c683a44548038a1b585b84e94 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 5328cf9c683a44548038a1b585b84e94 already in progress: creating tablet
I20261120 03:00:47.721328  1059 tablet_service.cc:1542] Processing DeleteTablet for tablet 5328cf9c683a44548038a1b585b84e94 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:50744
W20261120 03:00:47.723109   486 catalog_manager.cc:5061] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 5328cf9c683a44548038a1b585b84e94 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 5328cf9c683a44548038a1b585b84e94 already in progress: creating tablet
I20261120 03:00:47.647679   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "742fab63984e4752ac6c62d32ae39ae7" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
W20261120 03:00:47.740890  1011 leader_election.cc:343] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:47.749367  1181 tablet_bootstrap.cc:492] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:47.749795  1181 ts_tablet_manager.cc:1449] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.204s	user 0.027s	sys 0.018s
I20261120 03:00:47.695385   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "742fab63984e4752ac6c62d32ae39ae7" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:47.752718  1181 raft_consensus.cc:366] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.753407  1181 raft_consensus.cc:392] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:47.753672  1181 raft_consensus.cc:787] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:47.750061   500 catalog_manager.cc:6146] Not sending DeleteTablet requests; no consensus state for tablet 89167f00787f44a989ffc2e960bd6abb
W20261120 03:00:47.754633  1012 leader_election.cc:343] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:47.755053  1012 leader_election.cc:304] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:47.756019  1189 raft_consensus.cc:366] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.756464  1190 raft_consensus.cc:2966] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:47.756826  1189 raft_consensus.cc:392] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:47.757129  1189 raft_consensus.cc:787] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:47.758033  1189 consensus_queue.cc:261] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.761399  1181 consensus_queue.cc:261] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.764134  1181 ts_tablet_manager.cc:1480] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.014s	user 0.005s	sys 0.002s
I20261120 03:00:47.765020  1181 tablet_bootstrap.cc:492] T 27cfe2018ee44ab7beaa59992dc6250a P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:47.768975  1180 raft_consensus.cc:366] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.769955  1180 raft_consensus.cc:392] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:47.770321  1180 raft_consensus.cc:787] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:47.771485  1180 consensus_queue.cc:261] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.775211  1181 tablet_bootstrap.cc:654] T 27cfe2018ee44ab7beaa59992dc6250a P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:47.781728  1189 ts_tablet_manager.cc:1480] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.110s	user 0.019s	sys 0.008s
I20261120 03:00:47.782855  1189 tablet_bootstrap.cc:492] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:47.784713   499 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_d57abf08233c42ae9477476dba610f1e" } new_table_name: "default.Table_23d44ffb5710432c998c27caba060a45" modify_external_catalogs: true
I20261120 03:00:47.792336  1189 tablet_bootstrap.cc:654] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:47.799008  1181 tablet_bootstrap.cc:492] T 27cfe2018ee44ab7beaa59992dc6250a P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:47.799495  1181 ts_tablet_manager.cc:1449] T 27cfe2018ee44ab7beaa59992dc6250a P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.035s	user 0.009s	sys 0.004s
I20261120 03:00:47.802450  1181 raft_consensus.cc:366] T 27cfe2018ee44ab7beaa59992dc6250a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.803246  1181 raft_consensus.cc:392] T 27cfe2018ee44ab7beaa59992dc6250a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:47.803548  1181 raft_consensus.cc:787] T 27cfe2018ee44ab7beaa59992dc6250a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:47.804441  1181 consensus_queue.cc:261] T 27cfe2018ee44ab7beaa59992dc6250a P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.806624  1181 ts_tablet_manager.cc:1480] T 27cfe2018ee44ab7beaa59992dc6250a P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.007s	user 0.006s	sys 0.002s
I20261120 03:00:47.807678  1181 tablet_bootstrap.cc:492] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
W20261120 03:00:47.815843   499 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 5f5c8711a6eb43c3aff1c27ed5115d6a has no consensus state
I20261120 03:00:47.820254  1180 ts_tablet_manager.cc:1480] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.234s	user 0.039s	sys 0.013s
I20261120 03:00:47.822724  1180 tablet_bootstrap.cc:492] T 742fab63984e4752ac6c62d32ae39ae7 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:47.822925   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (table default.Table_749e6064fff14577a78fa67002d19c78 [id=cfcd5fea97fb4ab3baff99eb28cd9476]). Expected version 1 got 0
I20261120 03:00:47.824747   501 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 742fab63984e4752ac6c62d32ae39ae7 (table default.Table_c0ee25431db34324af521703039717fe [id=bb08bbcb4357420398926675042c3cce]). Expected version 2 got 0
I20261120 03:00:47.825606   498 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (table default.Table_749e6064fff14577a78fa67002d19c78 [id=cfcd5fea97fb4ab3baff99eb28cd9476]). Expected version 1 got 0
W20261120 03:00:47.826937   502 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 1cf6d449fa2845a7b1f4cbd1de639d20 consensus state has no leader
I20261120 03:00:47.829195  1180 tablet_bootstrap.cc:654] T 742fab63984e4752ac6c62d32ae39ae7 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:47.834196   904 tablet_service.cc:1495] Processing CreateTablet for tablet 9f73b223756845bb8af7b7fd96c0f9fc (DEFAULT_TABLE table=default.Table_185133dc409b4978acdea3f7fd4c0774 [id=e5d7a83f0b454c539f39545c800f99ee]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.835489   904 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9f73b223756845bb8af7b7fd96c0f9fc. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:00:47.839861   825 tablet.cc:2554] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20261120 03:00:47.840863   498 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 742fab63984e4752ac6c62d32ae39ae7 (table default.Table_c0ee25431db34324af521703039717fe [id=bb08bbcb4357420398926675042c3cce]). Expected version 2 got 0
I20261120 03:00:47.842417   499 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_84d4c48133054235908348e9d8cf1f2a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:47.842990  1189 tablet_bootstrap.cc:492] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:47.843508  1189 ts_tablet_manager.cc:1449] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.061s	user 0.022s	sys 0.028s
W20261120 03:00:47.845104   499 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_84d4c48133054235908348e9d8cf1f2a in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:47.846848   757 tablet_service.cc:1495] Processing CreateTablet for tablet 9f73b223756845bb8af7b7fd96c0f9fc (DEFAULT_TABLE table=default.Table_185133dc409b4978acdea3f7fd4c0774 [id=e5d7a83f0b454c539f39545c800f99ee]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.846938  1059 tablet_service.cc:1495] Processing CreateTablet for tablet 9f73b223756845bb8af7b7fd96c0f9fc (DEFAULT_TABLE table=default.Table_185133dc409b4978acdea3f7fd4c0774 [id=e5d7a83f0b454c539f39545c800f99ee]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.847883  1180 tablet_bootstrap.cc:492] T 742fab63984e4752ac6c62d32ae39ae7 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:47.847935   757 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9f73b223756845bb8af7b7fd96c0f9fc. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.848120  1059 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9f73b223756845bb8af7b7fd96c0f9fc. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.848318  1180 ts_tablet_manager.cc:1449] T 742fab63984e4752ac6c62d32ae39ae7 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.026s	user 0.004s	sys 0.013s
I20261120 03:00:47.851109  1180 raft_consensus.cc:366] T 742fab63984e4752ac6c62d32ae39ae7 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.851778  1180 raft_consensus.cc:392] T 742fab63984e4752ac6c62d32ae39ae7 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:47.852056  1180 raft_consensus.cc:787] T 742fab63984e4752ac6c62d32ae39ae7 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:47.852248   502 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (table default.Table_34bfe641479b42ceac031dafd7e21d2d [id=cfcd5fea97fb4ab3baff99eb28cd9476]). Expected version 2 got 0
I20261120 03:00:47.852769  1180 consensus_queue.cc:261] T 742fab63984e4752ac6c62d32ae39ae7 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.855034   500 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_8ee8b078bd0440e392b0fc4dc992714c"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:47.857071   500 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_8ee8b078bd0440e392b0fc4dc992714c in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:47.860045  1180 ts_tablet_manager.cc:1480] T 742fab63984e4752ac6c62d32ae39ae7 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.011s	user 0.007s	sys 0.001s
W20261120 03:00:47.864339   965 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6960 after lost signal to thread 835
I20261120 03:00:47.864940   902 tablet_service.cc:1495] Processing CreateTablet for tablet 5f5c8711a6eb43c3aff1c27ed5115d6a (DEFAULT_TABLE table=default.Table_23d44ffb5710432c998c27caba060a45 [id=affba127653b4dcbac7fc9fe76ba797a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.865971  1180 tablet_bootstrap.cc:492] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:47.865984   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5f5c8711a6eb43c3aff1c27ed5115d6a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.866763  1189 raft_consensus.cc:366] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.867769  1190 raft_consensus.cc:534] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:47.868206  1190 raft_consensus.cc:556] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.867446  1189 raft_consensus.cc:392] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:47.870349  1189 raft_consensus.cc:787] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:47.871228   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1cf6d449fa2845a7b1f4cbd1de639d20" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:47.872354  1189 consensus_queue.cc:261] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.876664  1180 tablet_bootstrap.cc:654] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:47.877446  1190 leader_election.cc:290] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:47.877708   777 raft_consensus.cc:2062] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:00:47.878976   501 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_af067a13691947e1a29b9461e9fe301e"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:47.879240  1011 leader_election.cc:304] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:00:47.880434  1189 ts_tablet_manager.cc:1480] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.037s	user 0.006s	sys 0.000s
W20261120 03:00:47.880774   501 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_af067a13691947e1a29b9461e9fe301e in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:47.883474  1189 tablet_bootstrap.cc:492] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:47.881954   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1cf6d449fa2845a7b1f4cbd1de639d20" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:47.884541   924 raft_consensus.cc:2062] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:00:47.888031  1190 raft_consensus.cc:3024] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:47.888405  1190 raft_consensus.cc:534] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:47.888279   497 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_269fbc3b47a44895be4b7bebddeee096"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:47.888727  1190 raft_consensus.cc:3296] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
W20261120 03:00:47.890208   497 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_269fbc3b47a44895be4b7bebddeee096 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:47.891265  1189 tablet_bootstrap.cc:654] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:47.890087  1190 raft_consensus.cc:556] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.891755  1200 raft_consensus.cc:534] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:47.892186  1200 raft_consensus.cc:556] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.894343   496 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (table default.Table_34bfe641479b42ceac031dafd7e21d2d [id=cfcd5fea97fb4ab3baff99eb28cd9476]). Expected version 2 got 0
I20261120 03:00:47.906881   754 tablet_service.cc:1495] Processing CreateTablet for tablet 5f5c8711a6eb43c3aff1c27ed5115d6a (DEFAULT_TABLE table=default.Table_23d44ffb5710432c998c27caba060a45 [id=affba127653b4dcbac7fc9fe76ba797a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.907994   754 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5f5c8711a6eb43c3aff1c27ed5115d6a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.908237   496 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (table default.Table_34bfe641479b42ceac031dafd7e21d2d [id=cfcd5fea97fb4ab3baff99eb28cd9476]). Expected version 2 got 0
I20261120 03:00:47.917989   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1cf6d449fa2845a7b1f4cbd1de639d20" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:47.919153   777 raft_consensus.cc:2062] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:47.920909   857 leader_election.cc:304] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:47.923573  1059 tablet_service.cc:1542] Processing DeleteTablet for tablet 27cfe2018ee44ab7beaa59992dc6250a with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:47.924489  1186 tablet_replica.cc:333] T 27cfe2018ee44ab7beaa59992dc6250a P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:47.925379  1186 raft_consensus.cc:2440] T 27cfe2018ee44ab7beaa59992dc6250a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:47.925940  1186 raft_consensus.cc:2469] T 27cfe2018ee44ab7beaa59992dc6250a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:47.929216  1186 ts_tablet_manager.cc:1962] T 27cfe2018ee44ab7beaa59992dc6250a P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:47.929533   498 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (table default.Table_34bfe641479b42ceac031dafd7e21d2d [id=cfcd5fea97fb4ab3baff99eb28cd9476]). Expected version 2 got 0
I20261120 03:00:47.937790   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1cf6d449fa2845a7b1f4cbd1de639d20" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:47.938402   777 raft_consensus.cc:3296] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:47.938531  1059 tablet_service.cc:1495] Processing CreateTablet for tablet 5f5c8711a6eb43c3aff1c27ed5115d6a (DEFAULT_TABLE table=default.Table_23d44ffb5710432c998c27caba060a45 [id=affba127653b4dcbac7fc9fe76ba797a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:47.939153   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1cf6d449fa2845a7b1f4cbd1de639d20" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:47.939536   777 raft_consensus.cc:2062] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
I20261120 03:00:47.939661  1059 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5f5c8711a6eb43c3aff1c27ed5115d6a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:47.939693   924 raft_consensus.cc:3296] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:47.940558   502 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 742fab63984e4752ac6c62d32ae39ae7 (table default.Table_c0ee25431db34324af521703039717fe [id=bb08bbcb4357420398926675042c3cce]). Expected version 2 got 0
I20261120 03:00:47.942416  1190 leader_election.cc:290] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:47.945881   496 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_c0ee25431db34324af521703039717fe" } modify_external_catalogs: true
I20261120 03:00:47.947365  1011 leader_election.cc:304] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:00:47.948418  1190 raft_consensus.cc:3024] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:47.949167  1190 raft_consensus.cc:744] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Becoming Leader. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Running, Role: LEADER
I20261120 03:00:47.954674  1200 leader_election.cc:290] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:47.956326   496 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_c0ee25431db34324af521703039717fe" } modify_external_catalogs: true
I20261120 03:00:47.962641  1190 consensus_queue.cc:238] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.972597  1180 tablet_bootstrap.cc:492] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:47.973209  1180 ts_tablet_manager.cc:1449] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.107s	user 0.015s	sys 0.032s
I20261120 03:00:47.976853  1180 raft_consensus.cc:366] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.977856  1180 raft_consensus.cc:392] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:47.978305  1180 raft_consensus.cc:787] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:47.981149  1189 tablet_bootstrap.cc:492] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:47.981637  1189 ts_tablet_manager.cc:1449] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.098s	user 0.010s	sys 0.032s
I20261120 03:00:47.979480  1180 consensus_queue.cc:261] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.958235   502 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_091012122fb94f08a8a199b60c0b2f54" } modify_external_catalogs: true
I20261120 03:00:47.985270   502 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_091012122fb94f08a8a199b60c0b2f54" } modify_external_catalogs: true
I20261120 03:00:47.986840  1180 ts_tablet_manager.cc:1480] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.013s	user 0.005s	sys 0.001s
I20261120 03:00:47.985569  1189 raft_consensus.cc:366] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.987732  1189 raft_consensus.cc:392] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:47.988065  1189 raft_consensus.cc:787] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
W20261120 03:00:47.989375   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 68757.9us
I20261120 03:00:47.991852  1180 tablet_bootstrap.cc:492] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:47.989075  1189 consensus_queue.cc:261] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:47.999063  1198 raft_consensus.cc:534] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:47.999557  1198 raft_consensus.cc:556] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.002609  1180 tablet_bootstrap.cc:654] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:48.002668  1198 leader_election.cc:290] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:48.020993  1180 tablet_bootstrap.cc:492] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:48.022533  1180 ts_tablet_manager.cc:1449] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.031s	user 0.000s	sys 0.012s
I20261120 03:00:48.033213   924 raft_consensus.cc:2062] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
I20261120 03:00:48.033257  1200 raft_consensus.cc:2984] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader pre-election decision vote started in defunct term 0: won
I20261120 03:00:48.025738  1180 raft_consensus.cc:366] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.035013  1180 raft_consensus.cc:392] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:48.035375  1180 raft_consensus.cc:787] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:48.036062  1180 consensus_queue.cc:261] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.037039   501 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_23d44ffb5710432c998c27caba060a45" } new_table_name: "default.Table_0a4ee63ac2c442c5bae84e199b8a9664" modify_external_catalogs: true
I20261120 03:00:48.040791  1180 ts_tablet_manager.cc:1480] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.017s	user 0.005s	sys 0.001s
I20261120 03:00:48.041708  1180 tablet_bootstrap.cc:492] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:48.045193   754 tablet_service.cc:1542] Processing DeleteTablet for tablet 27cfe2018ee44ab7beaa59992dc6250a with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:52644
W20261120 03:00:48.049659   486 catalog_manager.cc:5061] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 27cfe2018ee44ab7beaa59992dc6250a because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 27cfe2018ee44ab7beaa59992dc6250a already in progress: creating tablet
I20261120 03:00:48.051626   499 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_828236dc53434d038b86eb38ebacf7c6" } new_table_name: "default.Table_6eac430dad15473f8cb647b923b0df15" modify_external_catalogs: true
I20261120 03:00:48.054520  1180 tablet_bootstrap.cc:654] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:48.064989  1079 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1cf6d449fa2845a7b1f4cbd1de639d20" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:48.068147   976 tablet.cc:2554] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20261120 03:00:48.094331   496 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 742fab63984e4752ac6c62d32ae39ae7
I20261120 03:00:48.092936   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "742fab63984e4752ac6c62d32ae39ae7" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:48.098420   923 raft_consensus.cc:2062] T 742fab63984e4752ac6c62d32ae39ae7 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:48.098824  1059 tablet_service.cc:1542] Processing DeleteTablet for tablet 742fab63984e4752ac6c62d32ae39ae7 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:48.099531   754 tablet_service.cc:1542] Processing DeleteTablet for tablet 742fab63984e4752ac6c62d32ae39ae7 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:48.099936   710 leader_election.cc:304] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:48.100224   902 tablet_service.cc:1542] Processing DeleteTablet for tablet 742fab63984e4752ac6c62d32ae39ae7 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:48.101078  1185 tablet_replica.cc:333] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:48.101508  1209 raft_consensus.cc:3024] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:48.101892  1209 raft_consensus.cc:534] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:48.102243  1209 raft_consensus.cc:3296] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:48.103209  1189 ts_tablet_manager.cc:1480] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.121s	user 0.025s	sys 0.008s
I20261120 03:00:48.104133  1189 tablet_bootstrap.cc:492] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:48.113880  1189 tablet_bootstrap.cc:654] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:48.115890  1186 ts_tablet_manager.cc:1975] T 27cfe2018ee44ab7beaa59992dc6250a P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:48.116315  1186 log.cc:1199] T 27cfe2018ee44ab7beaa59992dc6250a P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/27cfe2018ee44ab7beaa59992dc6250a
I20261120 03:00:48.117208  1186 ts_tablet_manager.cc:1996] T 27cfe2018ee44ab7beaa59992dc6250a P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:48.065990  1181 tablet_bootstrap.cc:654] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:48.128916  1184 tablet_replica.cc:333] T 742fab63984e4752ac6c62d32ae39ae7 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:48.129910  1184 raft_consensus.cc:2440] T 742fab63984e4752ac6c62d32ae39ae7 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:48.130576  1184 raft_consensus.cc:2469] T 742fab63984e4752ac6c62d32ae39ae7 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:48.132272  1200 raft_consensus.cc:534] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:48.132763  1200 raft_consensus.cc:556] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.134913  1200 leader_election.cc:290] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:48.138253  1189 tablet_bootstrap.cc:492] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:48.138711  1189 ts_tablet_manager.cc:1449] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.035s	user 0.018s	sys 0.004s
I20261120 03:00:48.138832  1186 tablet_replica.cc:333] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:48.140301  1184 ts_tablet_manager.cc:1962] T 742fab63984e4752ac6c62d32ae39ae7 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:48.146098  1079 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5328cf9c683a44548038a1b585b84e94" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:48.152997  1186 raft_consensus.cc:2440] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:48.153520  1186 raft_consensus.cc:2469] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus is shut down!
W20261120 03:00:48.153843   857 leader_election.cc:343] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:48.146907  1188 consensus_queue.cc:1049] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20261120 03:00:48.155467  1186 ts_tablet_manager.cc:1962] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:48.156948  1188 consensus_queue.cc:1049] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:48.179934   497 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_d942907f0c164ec88eafa6129c513f31"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:48.181738   497 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d942907f0c164ec88eafa6129c513f31 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:00:48.186527   486 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.126:44467, user_credentials={real_user=slave}} blocked reactor thread for 79493.4us
I20261120 03:00:48.187361   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 27cfe2018ee44ab7beaa59992dc6250a (table default.Table_9b27b75009eb476c9d595545e2c25b55 [id=3937e02f9ba6411fbf5e91e2cdb235d3]) successfully deleted
W20261120 03:00:48.192116   501 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 5f5c8711a6eb43c3aff1c27ed5115d6a consensus state has no leader
I20261120 03:00:48.198486  1189 raft_consensus.cc:366] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.199170  1189 raft_consensus.cc:392] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:48.199440  1189 raft_consensus.cc:787] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:48.200074  1189 consensus_queue.cc:261] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.203462  1198 raft_consensus.cc:534] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:48.203896  1198 raft_consensus.cc:556] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.205778  1198 leader_election.cc:290] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:48.208873  1186 ts_tablet_manager.cc:1975] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:48.209272  1186 log.cc:1199] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/742fab63984e4752ac6c62d32ae39ae7
I20261120 03:00:48.212589  1209 raft_consensus.cc:556] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.213657  1185 raft_consensus.cc:2440] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:48.216424  1185 raft_consensus.cc:2469] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:48.209921  1186 ts_tablet_manager.cc:1996] T 742fab63984e4752ac6c62d32ae39ae7 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:48.230787   754 tablet_service.cc:1542] Processing DeleteTablet for tablet 27cfe2018ee44ab7beaa59992dc6250a with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:48.232235   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 742fab63984e4752ac6c62d32ae39ae7 (table default.Table_c0ee25431db34324af521703039717fe [id=bb08bbcb4357420398926675042c3cce]) successfully deleted
I20261120 03:00:48.232718  1189 ts_tablet_manager.cc:1480] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.094s	user 0.007s	sys 0.009s
I20261120 03:00:48.233309   502 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet a48ff491d14e40b3ba50bf5bde1757f0
I20261120 03:00:48.233682  1189 tablet_bootstrap.cc:492] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:48.235769   754 tablet_service.cc:1542] Processing DeleteTablet for tablet a48ff491d14e40b3ba50bf5bde1757f0 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:48.237242  1059 tablet_service.cc:1542] Processing DeleteTablet for tablet a48ff491d14e40b3ba50bf5bde1757f0 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:48.237427   502 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_6765bb99ca544b0886f578b8f98d2a61"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:48.238832   486 catalog_manager.cc:5061] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet a48ff491d14e40b3ba50bf5bde1757f0 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet a48ff491d14e40b3ba50bf5bde1757f0 already in progress: creating tablet
W20261120 03:00:48.239843   502 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_6765bb99ca544b0886f578b8f98d2a61 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:00:48.241155   499 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet ec08ada9d82b4692a496fef3f8c53807 consensus state has no leader
I20261120 03:00:48.242147  1209 leader_election.cc:290] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:48.246057   498 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_d2ca6a35f8b842d784167fd22339644a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:48.247047   902 tablet_service.cc:1542] Processing DeleteTablet for tablet a48ff491d14e40b3ba50bf5bde1757f0 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:44896
W20261120 03:00:48.247758   498 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d2ca6a35f8b842d784167fd22339644a in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:48.257350   499 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_b901331d0fdd40a89231797be95dd176"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:48.258503  1189 tablet_bootstrap.cc:654] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:48.259025   499 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_b901331d0fdd40a89231797be95dd176 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:48.288367   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5328cf9c683a44548038a1b585b84e94" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
W20261120 03:00:48.291154   709 leader_election.cc:343] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Not found: Tablet not found: 742fab63984e4752ac6c62d32ae39ae7
W20261120 03:00:48.294338   857 leader_election.cc:343] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:48.295291   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "27cfe2018ee44ab7beaa59992dc6250a" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:48.294839   857 leader_election.cc:304] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:48.299992  1181 tablet_bootstrap.cc:492] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:48.300678  1206 raft_consensus.cc:2966] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:48.300781   497 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_84d4c48133054235908348e9d8cf1f2a" } modify_external_catalogs: true
I20261120 03:00:48.296973   922 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "742fab63984e4752ac6c62d32ae39ae7" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:48.301301   497 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_84d4c48133054235908348e9d8cf1f2a" } modify_external_catalogs: true
W20261120 03:00:48.309526   710 leader_election.cc:343] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: Tablet not RUNNING: STOPPED
I20261120 03:00:48.310521  1185 ts_tablet_manager.cc:1962] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
W20261120 03:00:48.321424   487 catalog_manager.cc:4863] Async tablet task AlterTable RPC for tablet ec08ada9d82b4692a496fef3f8c53807 (table default.Table_6eac430dad15473f8cb647b923b0df15 [id=c4ac173f0a0e41df96cc9da60adc2ab1], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet ec08ada9d82b4692a496fef3f8c53807 consensus state has no leader
I20261120 03:00:48.327280  1180 tablet_bootstrap.cc:492] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:48.327884  1180 ts_tablet_manager.cc:1449] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.286s	user 0.025s	sys 0.031s
I20261120 03:00:48.331524  1180 raft_consensus.cc:366] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
W20261120 03:00:48.332752   486 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.124:38133, user_credentials={real_user=slave}} blocked reactor thread for 44301.5us
I20261120 03:00:48.351521  1181 ts_tablet_manager.cc:1449] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.544s	user 0.008s	sys 0.004s
I20261120 03:00:48.355365   500 catalog_manager.cc:5856] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 reported cstate change: term changed from 0 to 1, leader changed from <none> to 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67). New cstate: current_term: 1 leader_uuid: "823abdbd90ba47bbacad0f8dcff78234" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: HEALTHY } } }
I20261120 03:00:48.357393   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (table default.Table_34bfe641479b42ceac031dafd7e21d2d [id=cfcd5fea97fb4ab3baff99eb28cd9476]). Expected version 2 got 0
W20261120 03:00:48.292371   709 leader_election.cc:343] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Not found: Tablet not found: 27cfe2018ee44ab7beaa59992dc6250a
W20261120 03:00:48.358464   710 leader_election.cc:343] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:48.360481  1180 raft_consensus.cc:392] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:48.360772  1180 raft_consensus.cc:787] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:48.360558   710 leader_election.cc:304] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:48.360107  1184 ts_tablet_manager.cc:1975] T 742fab63984e4752ac6c62d32ae39ae7 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
W20261120 03:00:48.361570   710 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 51223.6us
I20261120 03:00:48.361972  1184 log.cc:1199] T 742fab63984e4752ac6c62d32ae39ae7 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/742fab63984e4752ac6c62d32ae39ae7
I20261120 03:00:48.364616  1219 raft_consensus.cc:2966] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20261120 03:00:48.366242   709 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 73905.9us
I20261120 03:00:48.370963  1184 ts_tablet_manager.cc:1996] T 742fab63984e4752ac6c62d32ae39ae7 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
W20261120 03:00:48.374514   709 leader_election.cc:343] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Not found: Tablet not found: 742fab63984e4752ac6c62d32ae39ae7
I20261120 03:00:48.371712  1180 consensus_queue.cc:261] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.374989   709 leader_election.cc:304] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:48.380573   904 tablet_service.cc:1495] Processing CreateTablet for tablet c3b56d0ccab140c28a37808d9d5902c6 (DEFAULT_TABLE table=default.Table_84d4c48133054235908348e9d8cf1f2a [id=5312622ebb1347f78430eab828632d38]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:48.381705   904 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c3b56d0ccab140c28a37808d9d5902c6. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:48.384112   754 tablet_service.cc:1495] Processing CreateTablet for tablet c3b56d0ccab140c28a37808d9d5902c6 (DEFAULT_TABLE table=default.Table_84d4c48133054235908348e9d8cf1f2a [id=5312622ebb1347f78430eab828632d38]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:48.385764   754 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c3b56d0ccab140c28a37808d9d5902c6. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:48.390425  1181 raft_consensus.cc:366] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.391212  1181 raft_consensus.cc:392] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:48.396329  1181 raft_consensus.cc:787] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:48.399029  1180 ts_tablet_manager.cc:1480] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.071s	user 0.007s	sys 0.001s
I20261120 03:00:48.406266  1059 tablet_service.cc:1495] Processing CreateTablet for tablet c3b56d0ccab140c28a37808d9d5902c6 (DEFAULT_TABLE table=default.Table_84d4c48133054235908348e9d8cf1f2a [id=5312622ebb1347f78430eab828632d38]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:48.407389  1059 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c3b56d0ccab140c28a37808d9d5902c6. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:48.408663   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 742fab63984e4752ac6c62d32ae39ae7 (table default.Table_c0ee25431db34324af521703039717fe [id=bb08bbcb4357420398926675042c3cce]) successfully deleted
I20261120 03:00:48.409969  1180 tablet_bootstrap.cc:492] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
W20261120 03:00:48.411793   487 catalog_manager.cc:5061] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet a48ff491d14e40b3ba50bf5bde1757f0 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet a48ff491d14e40b3ba50bf5bde1757f0 already in progress: creating tablet
I20261120 03:00:48.418715  1181 consensus_queue.cc:261] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
W20261120 03:00:48.375735   709 raft_consensus.cc:2902] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6: unable to run election callback: Service unavailable: Thread pool token was shut down
W20261120 03:00:48.420608   709 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 46150.7us
I20261120 03:00:48.424985  1185 ts_tablet_manager.cc:1975] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:48.425465  1185 log.cc:1199] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/742fab63984e4752ac6c62d32ae39ae7
I20261120 03:00:48.434665   499 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_8ee8b078bd0440e392b0fc4dc992714c" } new_table_name: "default.Table_91d5519024c0442b9cbce1cf67444e6e" modify_external_catalogs: true
I20261120 03:00:48.441501   495 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_6eac430dad15473f8cb647b923b0df15" } modify_external_catalogs: true
I20261120 03:00:48.442044   495 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_6eac430dad15473f8cb647b923b0df15" } modify_external_catalogs: true
I20261120 03:00:48.443481  1181 ts_tablet_manager.cc:1480] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.092s	user 0.007s	sys 0.000s
I20261120 03:00:48.455142  1180 tablet_bootstrap.cc:654] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:48.457159   501 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_d942907f0c164ec88eafa6129c513f31" } new_table_name: "default.Table_9710f1c011fb4249b18b31e6d859dd3a" modify_external_catalogs: true
I20261120 03:00:48.457989  1189 tablet_bootstrap.cc:492] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:48.458591  1189 ts_tablet_manager.cc:1449] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.225s	user 0.005s	sys 0.015s
I20261120 03:00:48.460678  1229 raft_consensus.cc:534] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:48.461232  1229 raft_consensus.cc:556] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.462867  1219 raft_consensus.cc:534] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:48.463369  1219 raft_consensus.cc:556] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.466892  1219 leader_election.cc:290] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:48.471671  1229 leader_election.cc:290] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:48.471619   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ec08ada9d82b4692a496fef3f8c53807" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:48.472355   924 raft_consensus.cc:2062] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:48.473356   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ec08ada9d82b4692a496fef3f8c53807" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:48.473457   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ec08ada9d82b4692a496fef3f8c53807" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:48.474268   777 raft_consensus.cc:2062] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:00:48.473963   924 raft_consensus.cc:2062] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:00:48.475417  1012 leader_election.cc:304] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:00:48.476812   710 leader_election.cc:304] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:48.478917  1219 raft_consensus.cc:3024] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:48.479429  1219 raft_consensus.cc:534] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:48.479815  1219 raft_consensus.cc:3296] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
W20261120 03:00:48.487372   499 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 06c3bac919e342b09dd6af13aebc387b has no consensus state
I20261120 03:00:48.489075   497 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet c3b56d0ccab140c28a37808d9d5902c6
I20261120 03:00:48.489254  1219 raft_consensus.cc:556] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.493444   904 tablet_service.cc:1542] Processing DeleteTablet for tablet c3b56d0ccab140c28a37808d9d5902c6 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:48.493341   496 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (table default.Table_34bfe641479b42ceac031dafd7e21d2d [id=cfcd5fea97fb4ab3baff99eb28cd9476]). Expected version 2 got 0
I20261120 03:00:48.493589   497 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_e07b2ba4d9864d148d7088622b10e03c"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:48.495746   497 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_e07b2ba4d9864d148d7088622b10e03c in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:00:48.495821   487 catalog_manager.cc:5061] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet c3b56d0ccab140c28a37808d9d5902c6 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet c3b56d0ccab140c28a37808d9d5902c6 already in progress: creating tablet
I20261120 03:00:48.494107   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ec08ada9d82b4692a496fef3f8c53807" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:48.496711   924 raft_consensus.cc:3296] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:48.497548   502 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (table default.Table_34bfe641479b42ceac031dafd7e21d2d [id=cfcd5fea97fb4ab3baff99eb28cd9476]). Expected version 2 got 0
I20261120 03:00:48.498003   757 tablet_service.cc:1542] Processing DeleteTablet for tablet c3b56d0ccab140c28a37808d9d5902c6 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:48.498874  1056 tablet_service.cc:1542] Processing DeleteTablet for tablet c3b56d0ccab140c28a37808d9d5902c6 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:48.492165  1219 leader_election.cc:290] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:48.501713   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (table default.Table_34bfe641479b42ceac031dafd7e21d2d [id=cfcd5fea97fb4ab3baff99eb28cd9476]). Expected version 2 got 0
I20261120 03:00:48.501873  1229 raft_consensus.cc:3024] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:48.502327  1229 raft_consensus.cc:534] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:48.502615   495 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet ec08ada9d82b4692a496fef3f8c53807
I20261120 03:00:48.502661  1229 raft_consensus.cc:3296] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:48.504088   924 raft_consensus.cc:2062] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:48.505160   710 leader_election.cc:304] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:48.507591   904 tablet_service.cc:1542] Processing DeleteTablet for tablet ec08ada9d82b4692a496fef3f8c53807 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:48.508280  1184 tablet_replica.cc:333] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:48.508985  1184 raft_consensus.cc:2440] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:48.509441  1184 raft_consensus.cc:2469] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:48.511296  1184 ts_tablet_manager.cc:1962] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:48.511636   902 tablet_service.cc:1542] Processing DeleteTablet for tablet 5328cf9c683a44548038a1b585b84e94 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:48.514640   757 tablet_service.cc:1542] Processing DeleteTablet for tablet ec08ada9d82b4692a496fef3f8c53807 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:48.519119  1056 tablet_service.cc:1542] Processing DeleteTablet for tablet ec08ada9d82b4692a496fef3f8c53807 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:48.520066  1186 tablet_replica.cc:333] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:48.524278  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ec08ada9d82b4692a496fef3f8c53807" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:48.526324   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet c3b56d0ccab140c28a37808d9d5902c6 because the tablet was not found. No further retry: Not found: Tablet not found: c3b56d0ccab140c28a37808d9d5902c6
I20261120 03:00:48.526256  1079 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ec08ada9d82b4692a496fef3f8c53807" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:48.530140  1229 raft_consensus.cc:556] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.531464  1078 raft_consensus.cc:2628] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5beeefe8bef54caeac463d05d8c30fa6 in current term 1: Already voted for candidate 823abdbd90ba47bbacad0f8dcff78234 in this term.
I20261120 03:00:48.532923  1186 raft_consensus.cc:2440] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:48.536418  1186 raft_consensus.cc:2469] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:48.537101  1219 raft_consensus.cc:3024] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:48.536866  1189 raft_consensus.cc:366] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.537779  1219 raft_consensus.cc:744] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
W20261120 03:00:48.542356   501 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet a642c32c1b6043978516d97efa2347a3 has no consensus state
I20261120 03:00:48.545197  1219 consensus_queue.cc:238] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.552238   501 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_dd2f7617ff0e401b9b862d5991110dd5"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:48.546693   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ec08ada9d82b4692a496fef3f8c53807" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:48.555646   496 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_07f0a3cd803f439889fe18fd21dfef3f"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:48.555826  1181 tablet_bootstrap.cc:492] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
W20261120 03:00:48.557473   496 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_07f0a3cd803f439889fe18fd21dfef3f in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:48.562646  1185 ts_tablet_manager.cc:1996] T 742fab63984e4752ac6c62d32ae39ae7 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:48.563828  1180 tablet_bootstrap.cc:492] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
W20261120 03:00:48.564256   501 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_dd2f7617ff0e401b9b862d5991110dd5 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:48.564302  1180 ts_tablet_manager.cc:1449] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.158s	user 0.011s	sys 0.003s
I20261120 03:00:48.567011   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ec08ada9d82b4692a496fef3f8c53807" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
W20261120 03:00:48.568394  1012 leader_election.cc:343] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: Tablet not RUNNING: STOPPED
I20261120 03:00:48.568887  1012 leader_election.cc:304] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
W20261120 03:00:48.569581  1012 raft_consensus.cc:2902] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234: unable to run election callback: Service unavailable: Thread pool token was shut down
I20261120 03:00:48.570315  1189 raft_consensus.cc:392] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:48.570734  1189 raft_consensus.cc:787] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:48.571167  1229 leader_election.cc:290] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:48.571485  1180 raft_consensus.cc:366] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.572170  1180 raft_consensus.cc:392] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:48.572451  1180 raft_consensus.cc:787] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:48.572026  1189 consensus_queue.cc:261] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.541705   500 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (table default.Table_34bfe641479b42ceac031dafd7e21d2d [id=cfcd5fea97fb4ab3baff99eb28cd9476]). Expected version 2 got 0
I20261120 03:00:48.576674  1186 ts_tablet_manager.cc:1962] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:48.581377   502 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (table default.Table_34bfe641479b42ceac031dafd7e21d2d [id=cfcd5fea97fb4ab3baff99eb28cd9476]). Expected version 2 got 0
I20261120 03:00:48.588759   904 tablet_service.cc:1542] Processing DeleteTablet for tablet 27cfe2018ee44ab7beaa59992dc6250a with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:48.591514  1180 consensus_queue.cc:261] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
W20261120 03:00:48.595388   710 consensus_peers.cc:597] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Couldn't send request to peer 3d30c2428730481db9b0ffd279dd3440. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:48.601611  1180 ts_tablet_manager.cc:1480] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.036s	user 0.006s	sys 0.002s
I20261120 03:00:48.594916  1184 ts_tablet_manager.cc:1975] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:48.608563  1184 log.cc:1199] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/ec08ada9d82b4692a496fef3f8c53807
I20261120 03:00:48.609647  1184 ts_tablet_manager.cc:1996] T ec08ada9d82b4692a496fef3f8c53807 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:48.610549  1206 raft_consensus.cc:534] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:48.610927  1206 raft_consensus.cc:556] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.613276  1206 leader_election.cc:290] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:48.613862   904 tablet_service.cc:1542] Processing DeleteTablet for tablet a48ff491d14e40b3ba50bf5bde1757f0 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:48.608971  1180 tablet_bootstrap.cc:492] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:48.615923   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a48ff491d14e40b3ba50bf5bde1757f0" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
W20261120 03:00:48.617450   857 leader_election.cc:343] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:48.617367  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a48ff491d14e40b3ba50bf5bde1757f0" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:48.622583   857 leader_election.cc:343] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:48.623081   857 leader_election.cc:304] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:48.624531  1206 raft_consensus.cc:2966] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:48.628095   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet ec08ada9d82b4692a496fef3f8c53807 (table default.Table_6eac430dad15473f8cb647b923b0df15 [id=c4ac173f0a0e41df96cc9da60adc2ab1]) successfully deleted
I20261120 03:00:48.631174  1184 tablet_replica.cc:333] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:48.635901  1184 raft_consensus.cc:2440] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:48.636641  1184 raft_consensus.cc:2469] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:48.639108  1184 ts_tablet_manager.cc:1962] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:48.642879  1189 ts_tablet_manager.cc:1480] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.184s	user 0.004s	sys 0.003s
I20261120 03:00:48.644016  1189 tablet_bootstrap.cc:492] T a48ff491d14e40b3ba50bf5bde1757f0 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:48.635056  1180 tablet_bootstrap.cc:654] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:48.653355  1206 raft_consensus.cc:534] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:48.653885  1206 raft_consensus.cc:556] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.655921  1206 leader_election.cc:290] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:48.657536   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "27cfe2018ee44ab7beaa59992dc6250a" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:48.658401   777 raft_consensus.cc:2062] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:48.659634   857 leader_election.cc:304] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:48.659578   501 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_d52207672f8f40acb7026fec3eb73d85"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:48.660655  1206 raft_consensus.cc:3024] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:48.661046  1206 raft_consensus.cc:534] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
W20261120 03:00:48.661322   501 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d52207672f8f40acb7026fec3eb73d85 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:48.661386  1206 raft_consensus.cc:3296] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
W20261120 03:00:48.665733   857 leader_election.cc:343] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Not found: Tablet not found: 27cfe2018ee44ab7beaa59992dc6250a
I20261120 03:00:48.667788  1180 tablet_bootstrap.cc:492] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:48.668249  1180 ts_tablet_manager.cc:1449] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.066s	user 0.010s	sys 0.001s
I20261120 03:00:48.668418   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 742fab63984e4752ac6c62d32ae39ae7 (table default.Table_c0ee25431db34324af521703039717fe [id=bb08bbcb4357420398926675042c3cce]) successfully deleted
I20261120 03:00:48.671103  1180 raft_consensus.cc:366] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.673219  1180 raft_consensus.cc:392] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:48.673571  1180 raft_consensus.cc:787] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:48.683321  1185 tablet_replica.cc:333] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:48.684911  1186 ts_tablet_manager.cc:1975] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:48.685269  1186 log.cc:1199] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/ec08ada9d82b4692a496fef3f8c53807
I20261120 03:00:48.685312  1185 raft_consensus.cc:2440] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:48.685817  1185 raft_consensus.cc:2469] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:48.686630  1180 consensus_queue.cc:261] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
W20261120 03:00:48.691600   709 consensus_peers.cc:597] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:48.698887   501 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_269fbc3b47a44895be4b7bebddeee096" } modify_external_catalogs: true
I20261120 03:00:48.699609  1186 ts_tablet_manager.cc:1996] T ec08ada9d82b4692a496fef3f8c53807 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:48.699527   501 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_269fbc3b47a44895be4b7bebddeee096" } modify_external_catalogs: true
I20261120 03:00:48.700730  1184 ts_tablet_manager.cc:1975] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:48.701015  1206 raft_consensus.cc:556] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.701597  1184 log.cc:1199] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/5328cf9c683a44548038a1b585b84e94
I20261120 03:00:48.702404   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet ec08ada9d82b4692a496fef3f8c53807 (table default.Table_6eac430dad15473f8cb647b923b0df15 [id=c4ac173f0a0e41df96cc9da60adc2ab1]) successfully deleted
I20261120 03:00:48.703184  1184 ts_tablet_manager.cc:1996] T 5328cf9c683a44548038a1b585b84e94 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:48.705969   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 5328cf9c683a44548038a1b585b84e94 (table default.Table_5e08feb4f2304718b4cc6c71a638e14c [id=59c24564b2eb44ccab8736e0ef5f733b]) successfully deleted
I20261120 03:00:48.708564   500 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_91d5519024c0442b9cbce1cf67444e6e" } modify_external_catalogs: true
I20261120 03:00:48.709151   500 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_91d5519024c0442b9cbce1cf67444e6e" } modify_external_catalogs: true
I20261120 03:00:48.709726  1181 tablet_bootstrap.cc:654] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:48.718331  1184 tablet_replica.cc:333] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:48.719898  1184 raft_consensus.cc:2440] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:48.721235  1184 raft_consensus.cc:2469] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:48.727684  1206 leader_election.cc:290] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
W20261120 03:00:48.731940   857 leader_election.cc:343] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Not found: Tablet not found: 27cfe2018ee44ab7beaa59992dc6250a
I20261120 03:00:48.746815  1181 tablet_bootstrap.cc:492] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:48.747233  1181 ts_tablet_manager.cc:1449] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.192s	user 0.013s	sys 0.000s
I20261120 03:00:48.748906  1184 ts_tablet_manager.cc:1962] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:48.750665  1181 raft_consensus.cc:366] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.751289  1181 raft_consensus.cc:392] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:48.751549  1181 raft_consensus.cc:787] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:48.752274  1181 consensus_queue.cc:261] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.755255  1181 ts_tablet_manager.cc:1480] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.008s	user 0.007s	sys 0.000s
I20261120 03:00:48.758211   496 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_9710f1c011fb4249b18b31e6d859dd3a" } new_table_name: "default.Table_e617c39b20b24b8f9477f625da46b661" modify_external_catalogs: true
I20261120 03:00:48.761320  1059 tablet_service.cc:1495] Processing CreateTablet for tablet a72d5d7896de4020a1ac4d2f8c551e15 (DEFAULT_TABLE table=default.Table_6765bb99ca544b0886f578b8f98d2a61 [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:48.762956  1181 tablet_bootstrap.cc:492] T a48ff491d14e40b3ba50bf5bde1757f0 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:48.762835   497 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_07f0a3cd803f439889fe18fd21dfef3f" } new_table_name: "default.Table_a1f866996679431e9e1c4c5f6f8cdd12" modify_external_catalogs: true
I20261120 03:00:48.764405   502 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 5f5c8711a6eb43c3aff1c27ed5115d6a (table default.Table_0a4ee63ac2c442c5bae84e199b8a9664 [id=affba127653b4dcbac7fc9fe76ba797a]). Expected version 2 got 0
I20261120 03:00:48.766391   501 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet d753dd170cab4f18980e85707c4093e9
I20261120 03:00:48.768903   904 tablet_service.cc:1542] Processing DeleteTablet for tablet d753dd170cab4f18980e85707c4093e9 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:48.772526  1056 tablet_service.cc:1542] Processing DeleteTablet for tablet d753dd170cab4f18980e85707c4093e9 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:48.773473  1059 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a72d5d7896de4020a1ac4d2f8c551e15. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:48.774950  1181 tablet_bootstrap.cc:654] T a48ff491d14e40b3ba50bf5bde1757f0 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:48.778463  1056 tablet_service.cc:1542] Processing DeleteTablet for tablet 5328cf9c683a44548038a1b585b84e94 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:48.787663   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "27cfe2018ee44ab7beaa59992dc6250a" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:48.789976  1189 tablet_bootstrap.cc:654] T a48ff491d14e40b3ba50bf5bde1757f0 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:48.791182   857 leader_election.cc:343] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:48.791648   857 leader_election.cc:304] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
W20261120 03:00:48.792248   857 raft_consensus.cc:2902] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440: unable to run election callback: Service unavailable: Thread pool token was shut down
I20261120 03:00:48.796283  1185 ts_tablet_manager.cc:1962] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:48.802796   757 tablet_service.cc:1542] Processing DeleteTablet for tablet d753dd170cab4f18980e85707c4093e9 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:48.802772   502 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_38dc7747f602474e93146d6065bd28bb"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:48.801900   754 tablet_service.cc:1542] Processing DeleteTablet for tablet 5328cf9c683a44548038a1b585b84e94 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:52644
W20261120 03:00:48.804294   502 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_38dc7747f602474e93146d6065bd28bb in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:00:48.805909   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet d753dd170cab4f18980e85707c4093e9 because the tablet was not found. No further retry: Not found: Tablet not found: d753dd170cab4f18980e85707c4093e9
I20261120 03:00:48.812240   500 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 06c3bac919e342b09dd6af13aebc387b
W20261120 03:00:48.814035   496 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet a642c32c1b6043978516d97efa2347a3 consensus state has no leader
I20261120 03:00:48.815460   904 tablet_service.cc:1542] Processing DeleteTablet for tablet 06c3bac919e342b09dd6af13aebc387b with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:48.816998  1184 ts_tablet_manager.cc:1975] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:48.817713  1184 log.cc:1199] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/27cfe2018ee44ab7beaa59992dc6250a
I20261120 03:00:48.819311  1209 raft_consensus.cc:534] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:48.819828  1209 raft_consensus.cc:556] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.821724  1209 leader_election.cc:290] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:48.819830  1189 tablet_bootstrap.cc:492] T a48ff491d14e40b3ba50bf5bde1757f0 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:48.822494  1189 ts_tablet_manager.cc:1449] T a48ff491d14e40b3ba50bf5bde1757f0 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.179s	user 0.006s	sys 0.007s
I20261120 03:00:48.823745  1056 tablet_service.cc:1542] Processing DeleteTablet for tablet 06c3bac919e342b09dd6af13aebc387b with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:48.825655  1189 raft_consensus.cc:366] T a48ff491d14e40b3ba50bf5bde1757f0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.826316  1189 raft_consensus.cc:392] T a48ff491d14e40b3ba50bf5bde1757f0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
W20261120 03:00:48.826269   710 leader_election.cc:343] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Not found: Tablet not found: 5328cf9c683a44548038a1b585b84e94
I20261120 03:00:48.826617  1189 raft_consensus.cc:787] T a48ff491d14e40b3ba50bf5bde1757f0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:48.827250  1189 consensus_queue.cc:261] T a48ff491d14e40b3ba50bf5bde1757f0 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.827782   496 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_0b2a37ca199f431fb72c9bb9db84eef3"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:48.829442   496 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_0b2a37ca199f431fb72c9bb9db84eef3 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:48.830050  1189 ts_tablet_manager.cc:1480] T a48ff491d14e40b3ba50bf5bde1757f0 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.007s	user 0.004s	sys 0.002s
I20261120 03:00:48.834990  1184 ts_tablet_manager.cc:1996] T 27cfe2018ee44ab7beaa59992dc6250a P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:48.835916  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5328cf9c683a44548038a1b585b84e94" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:48.836762  1078 raft_consensus.cc:2062] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:48.830915  1189 tablet_bootstrap.cc:492] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:48.840652   754 tablet_service.cc:1542] Processing DeleteTablet for tablet 06c3bac919e342b09dd6af13aebc387b with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:48.843775  1236 raft_consensus.cc:534] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:48.844269  1236 raft_consensus.cc:556] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.846426  1236 leader_election.cc:290] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:48.848356  1189 tablet_bootstrap.cc:654] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:48.847544   709 leader_election.cc:304] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440
I20261120 03:00:48.848906  1209 raft_consensus.cc:3024] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:48.849229  1209 raft_consensus.cc:534] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:48.849520  1209 raft_consensus.cc:3296] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:48.850226  1209 raft_consensus.cc:556] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.852841  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5f5c8711a6eb43c3aff1c27ed5115d6a" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:48.854677   857 leader_election.cc:343] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:48.854887   495 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_6723a1fea78c49ddbb3b7040faab3466"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:48.856809   495 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_6723a1fea78c49ddbb3b7040faab3466 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:48.858485  1209 leader_election.cc:290] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:48.858500   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5f5c8711a6eb43c3aff1c27ed5115d6a" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
W20261120 03:00:48.860222   710 leader_election.cc:343] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Not found: Tablet not found: 5328cf9c683a44548038a1b585b84e94
W20261120 03:00:48.861353   857 leader_election.cc:343] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:48.861792   857 leader_election.cc:304] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:48.862592  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5328cf9c683a44548038a1b585b84e94" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:48.862994  1236 raft_consensus.cc:2966] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:48.863222  1078 raft_consensus.cc:3296] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:48.864252  1078 raft_consensus.cc:2062] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:48.868641   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 27cfe2018ee44ab7beaa59992dc6250a (table default.Table_9b27b75009eb476c9d595545e2c25b55 [id=3937e02f9ba6411fbf5e91e2cdb235d3]) successfully deleted
I20261120 03:00:48.871543   709 leader_election.cc:304] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440
I20261120 03:00:48.872335  1221 raft_consensus.cc:3024] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:48.872749  1221 raft_consensus.cc:744] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:48.873497  1221 consensus_queue.cc:238] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.875265  1186 tablet_replica.cc:333] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:48.876019  1186 raft_consensus.cc:2440] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:48.876309  1181 tablet_bootstrap.cc:492] T a48ff491d14e40b3ba50bf5bde1757f0 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:48.876669  1186 raft_consensus.cc:2469] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:48.882018  1184 tablet_replica.cc:333] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:48.886237  1184 raft_consensus.cc:2440] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:48.886787  1184 raft_consensus.cc:2469] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:48.889233  1189 tablet_bootstrap.cc:492] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:48.889638  1189 ts_tablet_manager.cc:1449] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.059s	user 0.006s	sys 0.004s
I20261120 03:00:48.892112  1189 raft_consensus.cc:366] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.892752  1189 raft_consensus.cc:392] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:48.893019  1189 raft_consensus.cc:787] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:48.893338  1185 ts_tablet_manager.cc:1975] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:48.893740  1185 log.cc:1199] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/27cfe2018ee44ab7beaa59992dc6250a
I20261120 03:00:48.894814  1185 ts_tablet_manager.cc:1996] T 27cfe2018ee44ab7beaa59992dc6250a P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
W20261120 03:00:48.900081   709 consensus_peers.cc:597] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: STOPPING. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:48.900136   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 27cfe2018ee44ab7beaa59992dc6250a (table default.Table_9b27b75009eb476c9d595545e2c25b55 [id=3937e02f9ba6411fbf5e91e2cdb235d3]) successfully deleted
W20261120 03:00:48.903079   710 consensus_peers.cc:597] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Couldn't send request to peer 3d30c2428730481db9b0ffd279dd3440. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 5328cf9c683a44548038a1b585b84e94. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:48.903925  1184 ts_tablet_manager.cc:1962] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:48.905318  1180 ts_tablet_manager.cc:1480] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.237s	user 0.027s	sys 0.024s
I20261120 03:00:48.906225  1180 tablet_bootstrap.cc:492] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:48.911993  1180 tablet_bootstrap.cc:654] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:48.914996  1189 consensus_queue.cc:261] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.915577   904 tablet_service.cc:1495] Processing CreateTablet for tablet a72d5d7896de4020a1ac4d2f8c551e15 (DEFAULT_TABLE table=default.Table_6765bb99ca544b0886f578b8f98d2a61 [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:48.916635   904 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a72d5d7896de4020a1ac4d2f8c551e15. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:48.918516  1189 ts_tablet_manager.cc:1480] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.029s	user 0.006s	sys 0.001s
I20261120 03:00:48.919471  1189 tablet_bootstrap.cc:492] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:48.926923  1189 tablet_bootstrap.cc:654] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:48.937208  1189 tablet_bootstrap.cc:492] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:48.937614  1189 ts_tablet_manager.cc:1449] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.018s	user 0.004s	sys 0.006s
I20261120 03:00:48.939256   902 tablet_service.cc:1495] Processing CreateTablet for tablet a642c32c1b6043978516d97efa2347a3 (DEFAULT_TABLE table=default.Table_e617c39b20b24b8f9477f625da46b661 [id=2b97828a5a874a8cb13e283ef679c7fb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:48.940466   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a642c32c1b6043978516d97efa2347a3. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:48.942850  1185 tablet_replica.cc:333] T a48ff491d14e40b3ba50bf5bde1757f0 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:48.943639  1185 raft_consensus.cc:2440] T a48ff491d14e40b3ba50bf5bde1757f0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:48.944310  1185 raft_consensus.cc:2469] T a48ff491d14e40b3ba50bf5bde1757f0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:48.943011  1189 raft_consensus.cc:366] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.945374  1189 raft_consensus.cc:392] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:48.945622  1189 raft_consensus.cc:787] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:48.947365  1189 consensus_queue.cc:261] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.947796  1185 ts_tablet_manager.cc:1962] T a48ff491d14e40b3ba50bf5bde1757f0 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:48.950479  1189 ts_tablet_manager.cc:1480] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.013s	user 0.004s	sys 0.003s
I20261120 03:00:48.951277  1189 tablet_bootstrap.cc:492] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:48.958818  1059 tablet_service.cc:1495] Processing CreateTablet for tablet a642c32c1b6043978516d97efa2347a3 (DEFAULT_TABLE table=default.Table_e617c39b20b24b8f9477f625da46b661 [id=2b97828a5a874a8cb13e283ef679c7fb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:48.959913  1059 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a642c32c1b6043978516d97efa2347a3. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:48.961037   754 tablet_service.cc:1495] Processing CreateTablet for tablet a72d5d7896de4020a1ac4d2f8c551e15 (DEFAULT_TABLE table=default.Table_6765bb99ca544b0886f578b8f98d2a61 [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:48.962041   754 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a72d5d7896de4020a1ac4d2f8c551e15. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:48.963565  1189 tablet_bootstrap.cc:654] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:48.876739  1181 ts_tablet_manager.cc:1449] T a48ff491d14e40b3ba50bf5bde1757f0 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.114s	user 0.025s	sys 0.023s
I20261120 03:00:48.962791   757 tablet_service.cc:1495] Processing CreateTablet for tablet a642c32c1b6043978516d97efa2347a3 (DEFAULT_TABLE table=default.Table_e617c39b20b24b8f9477f625da46b661 [id=2b97828a5a874a8cb13e283ef679c7fb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:48.967243   757 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a642c32c1b6043978516d97efa2347a3. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:48.968456  1181 raft_consensus.cc:366] T a48ff491d14e40b3ba50bf5bde1757f0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.969100  1181 raft_consensus.cc:392] T a48ff491d14e40b3ba50bf5bde1757f0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:48.969362  1181 raft_consensus.cc:787] T a48ff491d14e40b3ba50bf5bde1757f0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:48.970345  1181 consensus_queue.cc:261] T a48ff491d14e40b3ba50bf5bde1757f0 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:48.971719   499 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 5f5c8711a6eb43c3aff1c27ed5115d6a (table default.Table_0a4ee63ac2c442c5bae84e199b8a9664 [id=affba127653b4dcbac7fc9fe76ba797a]). Expected version 2 got 0
I20261120 03:00:48.975548   502 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_e617c39b20b24b8f9477f625da46b661" } modify_external_catalogs: true
I20261120 03:00:48.976049   502 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_e617c39b20b24b8f9477f625da46b661" } modify_external_catalogs: true
I20261120 03:00:48.976560   500 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_69539e86f82e4df2a4b6888aa55293fc"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:48.977649  1181 ts_tablet_manager.cc:1480] T a48ff491d14e40b3ba50bf5bde1757f0 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.011s	user 0.003s	sys 0.002s
W20261120 03:00:48.978299   500 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_69539e86f82e4df2a4b6888aa55293fc in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:48.978590  1181 tablet_bootstrap.cc:492] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:48.985634  1186 ts_tablet_manager.cc:1962] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:48.986913  1056 tablet_service.cc:1542] Processing DeleteTablet for tablet a48ff491d14e40b3ba50bf5bde1757f0 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:48.997311   903 tablet_service.cc:1495] Processing CreateTablet for tablet d753dd170cab4f18980e85707c4093e9 (DEFAULT_TABLE table=default.Table_269fbc3b47a44895be4b7bebddeee096 [id=62fab6f5923b4aa3be379f3d3d8d3046]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:48.998485   903 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d753dd170cab4f18980e85707c4093e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:49.013191   753 tablet_service.cc:1495] Processing CreateTablet for tablet d753dd170cab4f18980e85707c4093e9 (DEFAULT_TABLE table=default.Table_269fbc3b47a44895be4b7bebddeee096 [id=62fab6f5923b4aa3be379f3d3d8d3046]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:49.014325   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d753dd170cab4f18980e85707c4093e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:49.011696  1056 tablet_service.cc:1495] Processing CreateTablet for tablet d753dd170cab4f18980e85707c4093e9 (DEFAULT_TABLE table=default.Table_269fbc3b47a44895be4b7bebddeee096 [id=62fab6f5923b4aa3be379f3d3d8d3046]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:49.019119  1056 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d753dd170cab4f18980e85707c4093e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:49.021975   496 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_185133dc409b4978acdea3f7fd4c0774" } new_table_name: "default.Table_97c478725540413b876cdcaad37d3f1d" modify_external_catalogs: true
I20261120 03:00:49.006062  1181 tablet_bootstrap.cc:654] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.026511   498 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_d2ca6a35f8b842d784167fd22339644a" } modify_external_catalogs: true
I20261120 03:00:49.027115   498 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_d2ca6a35f8b842d784167fd22339644a" } modify_external_catalogs: true
I20261120 03:00:49.072064  1058 tablet_service.cc:1495] Processing CreateTablet for tablet 1237b0d2fa414ef38271835b5d69d6b9 (DEFAULT_TABLE table=default.Table_af067a13691947e1a29b9461e9fe301e [id=93a0027d82064a208e2535349e9b042d]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:49.073173  1058 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1237b0d2fa414ef38271835b5d69d6b9. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:49.075353  1184 ts_tablet_manager.cc:1975] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:49.075731  1184 log.cc:1199] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/a48ff491d14e40b3ba50bf5bde1757f0
I20261120 03:00:49.076351  1184 ts_tablet_manager.cc:1996] T a48ff491d14e40b3ba50bf5bde1757f0 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:49.077162   755 tablet_service.cc:1495] Processing CreateTablet for tablet 1237b0d2fa414ef38271835b5d69d6b9 (DEFAULT_TABLE table=default.Table_af067a13691947e1a29b9461e9fe301e [id=93a0027d82064a208e2535349e9b042d]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:49.078254   755 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1237b0d2fa414ef38271835b5d69d6b9. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:49.080382  1180 tablet_bootstrap.cc:492] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:49.080394  1181 tablet_bootstrap.cc:492] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:49.080773  1181 ts_tablet_manager.cc:1449] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.102s	user 0.012s	sys 0.000s
I20261120 03:00:49.080770  1180 ts_tablet_manager.cc:1449] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.175s	user 0.032s	sys 0.017s
I20261120 03:00:49.081157  1186 ts_tablet_manager.cc:1975] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:49.081516  1186 log.cc:1199] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/5328cf9c683a44548038a1b585b84e94
I20261120 03:00:49.082245  1186 ts_tablet_manager.cc:1996] T 5328cf9c683a44548038a1b585b84e94 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:49.083040  1180 raft_consensus.cc:366] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.083236  1181 raft_consensus.cc:366] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.083643  1180 raft_consensus.cc:392] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.083897  1180 raft_consensus.cc:787] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.083817  1181 raft_consensus.cc:392] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.084519  1180 consensus_queue.cc:261] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.084863  1181 raft_consensus.cc:787] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.091765  1181 consensus_queue.cc:261] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.101171  1186 tablet_replica.cc:333] T a48ff491d14e40b3ba50bf5bde1757f0 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:49.101889  1186 raft_consensus.cc:2440] T a48ff491d14e40b3ba50bf5bde1757f0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:49.102378  1186 raft_consensus.cc:2469] T a48ff491d14e40b3ba50bf5bde1757f0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus is shut down!
W20261120 03:00:49.103497   497 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 5fee8dff291c43e9be0fb10b308718ce has no consensus state
I20261120 03:00:49.103489   502 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet a642c32c1b6043978516d97efa2347a3
I20261120 03:00:49.114408  1186 ts_tablet_manager.cc:1962] T a48ff491d14e40b3ba50bf5bde1757f0 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:49.115532  1056 tablet_service.cc:1542] Processing DeleteTablet for tablet a642c32c1b6043978516d97efa2347a3 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:49.116334   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 5328cf9c683a44548038a1b585b84e94 (table default.Table_5e08feb4f2304718b4cc6c71a638e14c [id=59c24564b2eb44ccab8736e0ef5f733b]) successfully deleted
W20261120 03:00:49.116948   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 06c3bac919e342b09dd6af13aebc387b because the tablet was not found. No further retry: Not found: Tablet not found: 06c3bac919e342b09dd6af13aebc387b
I20261120 03:00:49.121312  1180 ts_tablet_manager.cc:1480] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.040s	user 0.006s	sys 0.000s
I20261120 03:00:49.122200  1180 tablet_bootstrap.cc:492] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:49.122467  1181 ts_tablet_manager.cc:1480] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.041s	user 0.005s	sys 0.002s
I20261120 03:00:49.136967  1189 tablet_bootstrap.cc:492] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:49.137408  1189 ts_tablet_manager.cc:1449] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.186s	user 0.015s	sys 0.016s
I20261120 03:00:49.140130  1189 raft_consensus.cc:366] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.140738  1189 raft_consensus.cc:392] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.140985  1189 raft_consensus.cc:787] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.141654  1189 consensus_queue.cc:261] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.145860  1186 ts_tablet_manager.cc:1975] T a48ff491d14e40b3ba50bf5bde1757f0 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:49.152118  1253 tablet.cc:1872] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20261120 03:00:49.153046  1186 log.cc:1199] T a48ff491d14e40b3ba50bf5bde1757f0 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/a48ff491d14e40b3ba50bf5bde1757f0
I20261120 03:00:49.153672  1180 tablet_bootstrap.cc:654] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.153906  1186 ts_tablet_manager.cc:1996] T a48ff491d14e40b3ba50bf5bde1757f0 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:49.155055  1189 ts_tablet_manager.cc:1480] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.017s	user 0.005s	sys 0.000s
I20261120 03:00:49.155920  1189 tablet_bootstrap.cc:492] T a642c32c1b6043978516d97efa2347a3 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:49.163764  1237 raft_consensus.cc:534] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:49.164260  1237 raft_consensus.cc:556] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.167366  1181 tablet_bootstrap.cc:492] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:49.169788   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9f73b223756845bb8af7b7fd96c0f9fc" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:49.170543   777 raft_consensus.cc:2062] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:49.171070  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9f73b223756845bb8af7b7fd96c0f9fc" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:49.171777  1078 raft_consensus.cc:2062] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:49.171672   857 leader_election.cc:304] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:49.172439  1236 raft_consensus.cc:3024] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:49.172783  1236 raft_consensus.cc:534] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:49.173094  1236 raft_consensus.cc:3296] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:49.173825  1236 raft_consensus.cc:556] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.176642  1252 tablet.cc:1872] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20261120 03:00:49.179301   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet a48ff491d14e40b3ba50bf5bde1757f0 (table default.Table_091012122fb94f08a8a199b60c0b2f54 [id=17ac079fd9504c4285a664a9079f1205]) successfully deleted
W20261120 03:00:49.180495   487 catalog_manager.cc:5061] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet d753dd170cab4f18980e85707c4093e9 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet d753dd170cab4f18980e85707c4093e9 already in progress: creating tablet
W20261120 03:00:49.181177   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 06c3bac919e342b09dd6af13aebc387b because the tablet was not found. No further retry: Not found: Tablet not found: 06c3bac919e342b09dd6af13aebc387b
I20261120 03:00:49.187107   500 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_14745a85367e453183d6a0a197be3fbb"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:49.188867   500 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_14745a85367e453183d6a0a197be3fbb in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:49.190448  1221 raft_consensus.cc:534] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:49.190897  1221 raft_consensus.cc:556] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.192884  1221 leader_election.cc:290] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:49.193709   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "c3b56d0ccab140c28a37808d9d5902c6" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:49.195041  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "c3b56d0ccab140c28a37808d9d5902c6" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:49.195295   710 leader_election.cc:343] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:49.196040  1185 ts_tablet_manager.cc:1975] T a48ff491d14e40b3ba50bf5bde1757f0 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:49.196417  1185 log.cc:1199] T a48ff491d14e40b3ba50bf5bde1757f0 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/a48ff491d14e40b3ba50bf5bde1757f0
I20261120 03:00:49.197044  1185 ts_tablet_manager.cc:1996] T a48ff491d14e40b3ba50bf5bde1757f0 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
W20261120 03:00:49.197005   709 leader_election.cc:343] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:49.197417   709 leader_election.cc:304] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:49.198400  1221 raft_consensus.cc:2966] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:49.170907  1237 leader_election.cc:290] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:49.200606  1189 tablet_bootstrap.cc:654] T a642c32c1b6043978516d97efa2347a3 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.206218  1236 leader_election.cc:290] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:49.207525  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9f73b223756845bb8af7b7fd96c0f9fc" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:49.208081  1078 raft_consensus.cc:3296] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:49.208983  1078 raft_consensus.cc:2062] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:00:49.210117   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9f73b223756845bb8af7b7fd96c0f9fc" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:49.210642   777 raft_consensus.cc:3296] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:49.217761   777 raft_consensus.cc:2062] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:00:49.219375   857 leader_election.cc:304] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:49.219940  1181 tablet_bootstrap.cc:654] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.220316  1206 raft_consensus.cc:3024] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:49.220816  1206 raft_consensus.cc:744] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:00:49.221912  1206 consensus_queue.cc:238] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.225817   497 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_5203afc2eeba4e2abc5a94484597a25a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:49.227691   497 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_5203afc2eeba4e2abc5a94484597a25a in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:49.232650  1185 tablet_replica.cc:333] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:49.240594  1181 tablet_bootstrap.cc:492] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:49.240981  1181 ts_tablet_manager.cc:1449] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.074s	user 0.006s	sys 0.002s
I20261120 03:00:49.244442  1185 raft_consensus.cc:2440] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:49.244656   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet a48ff491d14e40b3ba50bf5bde1757f0 (table default.Table_091012122fb94f08a8a199b60c0b2f54 [id=17ac079fd9504c4285a664a9079f1205]) successfully deleted
I20261120 03:00:49.244925  1185 raft_consensus.cc:2469] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus is shut down!
W20261120 03:00:49.245366   486 catalog_manager.cc:5061] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet a642c32c1b6043978516d97efa2347a3 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet a642c32c1b6043978516d97efa2347a3 already in progress: creating tablet
I20261120 03:00:49.247354  1189 tablet_bootstrap.cc:492] T a642c32c1b6043978516d97efa2347a3 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:49.247749  1189 ts_tablet_manager.cc:1449] T a642c32c1b6043978516d97efa2347a3 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.092s	user 0.005s	sys 0.006s
I20261120 03:00:49.250283  1206 consensus_queue.cc:1049] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:49.251806  1236 consensus_queue.cc:1049] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:49.256704   904 tablet_service.cc:1495] Processing CreateTablet for tablet 1237b0d2fa414ef38271835b5d69d6b9 (DEFAULT_TABLE table=default.Table_af067a13691947e1a29b9461e9fe301e [id=93a0027d82064a208e2535349e9b042d]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:49.257938   904 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1237b0d2fa414ef38271835b5d69d6b9. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:49.266896   498 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 136056e5242a42ef8fa4b40c989b5423
I20261120 03:00:49.272840   902 tablet_service.cc:1542] Processing DeleteTablet for tablet 136056e5242a42ef8fa4b40c989b5423 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:49.276779  1185 ts_tablet_manager.cc:1962] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
W20261120 03:00:49.278100   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 136056e5242a42ef8fa4b40c989b5423 because the tablet was not found. No further retry: Not found: Tablet not found: 136056e5242a42ef8fa4b40c989b5423
I20261120 03:00:49.281946  1181 raft_consensus.cc:366] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.282655  1181 raft_consensus.cc:392] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.282912  1181 raft_consensus.cc:787] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.285109   903 tablet_service.cc:1542] Processing DeleteTablet for tablet a642c32c1b6043978516d97efa2347a3 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:44896
W20261120 03:00:49.287452   487 catalog_manager.cc:5061] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet a642c32c1b6043978516d97efa2347a3 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet a642c32c1b6043978516d97efa2347a3 already in progress: creating tablet
I20261120 03:00:49.287822  1189 raft_consensus.cc:366] T a642c32c1b6043978516d97efa2347a3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.293676  1181 consensus_queue.cc:261] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.288465  1189 raft_consensus.cc:392] T a642c32c1b6043978516d97efa2347a3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.295217  1189 raft_consensus.cc:787] T a642c32c1b6043978516d97efa2347a3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.295674   501 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 5f5c8711a6eb43c3aff1c27ed5115d6a (table default.Table_0a4ee63ac2c442c5bae84e199b8a9664 [id=affba127653b4dcbac7fc9fe76ba797a]). Expected version 2 got 0
W20261120 03:00:49.296841   496 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 9f73b223756845bb8af7b7fd96c0f9fc consensus state has no leader
I20261120 03:00:49.308225   502 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_ae4914f59f3847c8a2a0ef1d68a7f4b2"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:49.308712  1263 tablet.cc:1872] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
W20261120 03:00:49.309876   502 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_ae4914f59f3847c8a2a0ef1d68a7f4b2 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:49.314155   496 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_97c478725540413b876cdcaad37d3f1d" } modify_external_catalogs: true
I20261120 03:00:49.314383  1189 consensus_queue.cc:261] T a642c32c1b6043978516d97efa2347a3 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.314740   496 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_97c478725540413b876cdcaad37d3f1d" } modify_external_catalogs: true
I20261120 03:00:49.316299  1189 ts_tablet_manager.cc:1480] T a642c32c1b6043978516d97efa2347a3 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.068s	user 0.005s	sys 0.001s
I20261120 03:00:49.317119  1189 tablet_bootstrap.cc:492] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:49.319374  1181 ts_tablet_manager.cc:1480] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.078s	user 0.007s	sys 0.000s
I20261120 03:00:49.320340  1181 tablet_bootstrap.cc:492] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:49.325851   498 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_6765bb99ca544b0886f578b8f98d2a61" } new_table_name: "default.Table_f601bffe6ddb49b3a3c589ac7b25235c" modify_external_catalogs: true
I20261120 03:00:49.335868   754 tablet_service.cc:1542] Processing DeleteTablet for tablet a642c32c1b6043978516d97efa2347a3 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:49.338779   499 catalog_manager.cc:5856] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66). New cstate: current_term: 1 leader_uuid: "3d30c2428730481db9b0ffd279dd3440" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: HEALTHY } } }
I20261120 03:00:49.340456   499 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 9f73b223756845bb8af7b7fd96c0f9fc (table default.Table_97c478725540413b876cdcaad37d3f1d [id=e5d7a83f0b454c539f39545c800f99ee]). Expected version 1 got 0
W20261120 03:00:49.346766   486 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.124:38133, user_credentials={real_user=slave}} blocked reactor thread for 100581us
I20261120 03:00:49.347488   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet a48ff491d14e40b3ba50bf5bde1757f0 (table default.Table_091012122fb94f08a8a199b60c0b2f54 [id=17ac079fd9504c4285a664a9079f1205]) successfully deleted
I20261120 03:00:49.348599  1181 tablet_bootstrap.cc:654] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.359200  1189 tablet_bootstrap.cc:654] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.363623  1058 tablet_service.cc:1495] Processing CreateTablet for tablet 06c3bac919e342b09dd6af13aebc387b (DEFAULT_TABLE table=default.Table_91d5519024c0442b9cbce1cf67444e6e [id=927890104a3a4819878b6ae3f13c0e71]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:49.364750  1058 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 06c3bac919e342b09dd6af13aebc387b. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:49.365361   903 tablet_service.cc:1495] Processing CreateTablet for tablet 06c3bac919e342b09dd6af13aebc387b (DEFAULT_TABLE table=default.Table_91d5519024c0442b9cbce1cf67444e6e [id=927890104a3a4819878b6ae3f13c0e71]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:49.363829  1056 tablet_service.cc:1542] Processing DeleteTablet for tablet 136056e5242a42ef8fa4b40c989b5423 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:49.367592   754 tablet_service.cc:1542] Processing DeleteTablet for tablet 136056e5242a42ef8fa4b40c989b5423 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:49.370820   903 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 06c3bac919e342b09dd6af13aebc387b. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:49.353829  1185 ts_tablet_manager.cc:1975] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:49.370988   495 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_bc7d30a65be54db09b1378e3b21551f4"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:49.371384  1185 log.cc:1199] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/c3b56d0ccab140c28a37808d9d5902c6
I20261120 03:00:49.372102  1185 ts_tablet_manager.cc:1996] T c3b56d0ccab140c28a37808d9d5902c6 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
W20261120 03:00:49.373229   495 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_bc7d30a65be54db09b1378e3b21551f4 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:49.374954  1180 tablet_bootstrap.cc:492] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:49.375370  1180 ts_tablet_manager.cc:1449] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.253s	user 0.018s	sys 0.004s
I20261120 03:00:49.380183  1180 raft_consensus.cc:366] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.380806  1180 raft_consensus.cc:392] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.381057  1180 raft_consensus.cc:787] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.385931  1185 tablet_replica.cc:333] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:49.386767  1185 raft_consensus.cc:2440] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Raft consensus shutting down.
W20261120 03:00:49.389961   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 136056e5242a42ef8fa4b40c989b5423 because the tablet was not found. No further retry: Not found: Tablet not found: 136056e5242a42ef8fa4b40c989b5423
I20261120 03:00:49.390231   757 tablet_service.cc:1495] Processing CreateTablet for tablet 06c3bac919e342b09dd6af13aebc387b (DEFAULT_TABLE table=default.Table_91d5519024c0442b9cbce1cf67444e6e [id=927890104a3a4819878b6ae3f13c0e71]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:49.391181   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet c3b56d0ccab140c28a37808d9d5902c6 (table default.Table_84d4c48133054235908348e9d8cf1f2a [id=5312622ebb1347f78430eab828632d38]) successfully deleted
I20261120 03:00:49.391332   757 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 06c3bac919e342b09dd6af13aebc387b. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:49.396582   902 tablet_service.cc:1495] Processing CreateTablet for tablet 136056e5242a42ef8fa4b40c989b5423 (DEFAULT_TABLE table=default.Table_d2ca6a35f8b842d784167fd22339644a [id=afefca6af8d64812b4f3e37317a1c8d7]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:49.397030  1185 pending_rounds.cc:70] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6: Trying to abort 1 pending ops.
I20261120 03:00:49.397356  1185 pending_rounds.cc:77] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7352908384479977472 op_type: NO_OP noop_request { }
I20261120 03:00:49.397750  1185 raft_consensus.cc:3115] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
I20261120 03:00:49.397970   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 136056e5242a42ef8fa4b40c989b5423. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:49.398118  1185 raft_consensus.cc:2469] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:49.381688  1180 consensus_queue.cc:261] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.409646  1180 ts_tablet_manager.cc:1480] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.034s	user 0.006s	sys 0.001s
I20261120 03:00:49.410480  1180 tablet_bootstrap.cc:492] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:49.410821  1185 ts_tablet_manager.cc:1962] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:49.424424  1181 tablet_bootstrap.cc:492] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:49.424853  1181 ts_tablet_manager.cc:1449] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.105s	user 0.018s	sys 0.000s
I20261120 03:00:49.425812   901 tablet_service.cc:1495] Processing CreateTablet for tablet 3f65ba863447413cb122e728df3632fd (DEFAULT_TABLE table=default.Table_b901331d0fdd40a89231797be95dd176 [id=676ae8fc1a8c488d820a11b930b0ed5f]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:49.427016   901 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3f65ba863447413cb122e728df3632fd. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:49.428076  1181 raft_consensus.cc:366] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.428709  1181 raft_consensus.cc:392] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.428970  1181 raft_consensus.cc:787] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.435756  1189 tablet_bootstrap.cc:492] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:49.436465  1189 ts_tablet_manager.cc:1449] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.120s	user 0.014s	sys 0.008s
I20261120 03:00:49.438935  1189 raft_consensus.cc:366] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.439554  1189 raft_consensus.cc:392] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.439805  1189 raft_consensus.cc:787] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
W20261120 03:00:49.440706   489 catalog_manager.cc:4863] Async tablet task AlterTable RPC for tablet 5fee8dff291c43e9be0fb10b308718ce (table default.Table_a1f866996679431e9e1c4c5f6f8cdd12 [id=a275e215155b4824a1494f0954577fe8], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 5fee8dff291c43e9be0fb10b308718ce has no consensus state
I20261120 03:00:49.440454  1189 consensus_queue.cc:261] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.441313  1181 consensus_queue.cc:261] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.446110  1181 ts_tablet_manager.cc:1480] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.021s	user 0.007s	sys 0.000s
I20261120 03:00:49.446679  1180 tablet_bootstrap.cc:654] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.456797  1181 tablet_bootstrap.cc:492] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:49.462978  1189 ts_tablet_manager.cc:1480] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.026s	user 0.006s	sys 0.000s
I20261120 03:00:49.463884  1189 tablet_bootstrap.cc:492] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:49.465030  1181 tablet_bootstrap.cc:654] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.472960   497 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_38dc7747f602474e93146d6065bd28bb" } modify_external_catalogs: true
I20261120 03:00:49.473537   497 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_38dc7747f602474e93146d6065bd28bb" } modify_external_catalogs: true
W20261120 03:00:49.481195   486 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.126:44467, user_credentials={real_user=slave}} blocked reactor thread for 88968.7us
I20261120 03:00:49.481878  1185 ts_tablet_manager.cc:1975] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:49.482300  1185 log.cc:1199] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/ec08ada9d82b4692a496fef3f8c53807
I20261120 03:00:49.483017  1185 ts_tablet_manager.cc:1996] T ec08ada9d82b4692a496fef3f8c53807 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:49.484232  1189 tablet_bootstrap.cc:654] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.484071  1058 tablet_service.cc:1495] Processing CreateTablet for tablet 136056e5242a42ef8fa4b40c989b5423 (DEFAULT_TABLE table=default.Table_d2ca6a35f8b842d784167fd22339644a [id=afefca6af8d64812b4f3e37317a1c8d7]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:49.485359  1058 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 136056e5242a42ef8fa4b40c989b5423. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:49.500661  1056 tablet_service.cc:1495] Processing CreateTablet for tablet 3f65ba863447413cb122e728df3632fd (DEFAULT_TABLE table=default.Table_b901331d0fdd40a89231797be95dd176 [id=676ae8fc1a8c488d820a11b930b0ed5f]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:49.501761  1056 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3f65ba863447413cb122e728df3632fd. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:49.502373  1185 tablet_replica.cc:333] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:49.503123  1185 raft_consensus.cc:2440] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:49.503947  1185 pending_rounds.cc:70] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6: Trying to abort 1 pending ops.
I20261120 03:00:49.504199  1185 pending_rounds.cc:77] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7352908385808834560 op_type: NO_OP noop_request { }
I20261120 03:00:49.504534  1185 raft_consensus.cc:3115] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
I20261120 03:00:49.504853  1185 raft_consensus.cc:2469] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:49.507015  1181 tablet_bootstrap.cc:492] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:49.507411  1181 ts_tablet_manager.cc:1449] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.051s	user 0.008s	sys 0.002s
I20261120 03:00:49.507802  1189 tablet_bootstrap.cc:492] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:49.508188  1189 ts_tablet_manager.cc:1449] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.044s	user 0.008s	sys 0.001s
I20261120 03:00:49.508935   500 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_6723a1fea78c49ddbb3b7040faab3466" } new_table_name: "default.Table_3bb077af5e15449b85500aca3a1500c3" modify_external_catalogs: true
I20261120 03:00:49.509745  1181 raft_consensus.cc:366] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.513355  1181 raft_consensus.cc:392] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.513607  1181 raft_consensus.cc:787] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.513523  1189 raft_consensus.cc:366] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.514226  1181 consensus_queue.cc:261] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.518018  1189 raft_consensus.cc:392] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.518455  1189 raft_consensus.cc:787] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.519074  1189 consensus_queue.cc:261] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.520895  1189 ts_tablet_manager.cc:1480] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.012s	user 0.005s	sys 0.000s
I20261120 03:00:49.521700  1189 tablet_bootstrap.cc:492] T 06c3bac919e342b09dd6af13aebc387b P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:49.521802  1181 ts_tablet_manager.cc:1480] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.014s	user 0.006s	sys 0.000s
I20261120 03:00:49.527660   755 tablet_service.cc:1495] Processing CreateTablet for tablet 136056e5242a42ef8fa4b40c989b5423 (DEFAULT_TABLE table=default.Table_d2ca6a35f8b842d784167fd22339644a [id=afefca6af8d64812b4f3e37317a1c8d7]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:49.529333   755 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 136056e5242a42ef8fa4b40c989b5423. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:49.531651  1185 ts_tablet_manager.cc:1962] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:49.536149   757 tablet_service.cc:1495] Processing CreateTablet for tablet 3f65ba863447413cb122e728df3632fd (DEFAULT_TABLE table=default.Table_b901331d0fdd40a89231797be95dd176 [id=676ae8fc1a8c488d820a11b930b0ed5f]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:49.537246   757 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3f65ba863447413cb122e728df3632fd. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:49.544888  1181 tablet_bootstrap.cc:492] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:49.550575  1181 tablet_bootstrap.cc:654] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.555788  1189 tablet_bootstrap.cc:654] T 06c3bac919e342b09dd6af13aebc387b P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.563583   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet ec08ada9d82b4692a496fef3f8c53807 (table default.Table_6eac430dad15473f8cb647b923b0df15 [id=c4ac173f0a0e41df96cc9da60adc2ab1]) successfully deleted
W20261120 03:00:49.564364   486 catalog_manager.cc:5061] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet d753dd170cab4f18980e85707c4093e9 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet d753dd170cab4f18980e85707c4093e9 already in progress: creating tablet
I20261120 03:00:49.569437  1189 tablet_bootstrap.cc:492] T 06c3bac919e342b09dd6af13aebc387b P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:49.569839  1189 ts_tablet_manager.cc:1449] T 06c3bac919e342b09dd6af13aebc387b P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.048s	user 0.008s	sys 0.000s
I20261120 03:00:49.572067  1189 raft_consensus.cc:366] T 06c3bac919e342b09dd6af13aebc387b P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.572657  1189 raft_consensus.cc:392] T 06c3bac919e342b09dd6af13aebc387b P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.572902  1189 raft_consensus.cc:787] T 06c3bac919e342b09dd6af13aebc387b P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.581877  1181 tablet_bootstrap.cc:492] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:49.582325  1181 ts_tablet_manager.cc:1449] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.038s	user 0.011s	sys 0.000s
I20261120 03:00:49.586946  1180 tablet_bootstrap.cc:492] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:49.587342  1180 ts_tablet_manager.cc:1449] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.177s	user 0.020s	sys 0.011s
I20261120 03:00:49.589648  1180 raft_consensus.cc:366] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.590380  1180 raft_consensus.cc:392] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.590631  1180 raft_consensus.cc:787] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.591308  1180 consensus_queue.cc:261] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.584679  1181 raft_consensus.cc:366] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.593257  1181 raft_consensus.cc:392] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.593514  1181 raft_consensus.cc:787] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.594208  1181 consensus_queue.cc:261] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.596697  1180 ts_tablet_manager.cc:1480] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.009s	user 0.006s	sys 0.000s
I20261120 03:00:49.597512  1180 tablet_bootstrap.cc:492] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:49.606021  1181 ts_tablet_manager.cc:1480] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.023s	user 0.005s	sys 0.000s
I20261120 03:00:49.606853  1181 tablet_bootstrap.cc:492] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:49.610136  1185 ts_tablet_manager.cc:1975] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:49.610549  1185 log.cc:1199] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/5328cf9c683a44548038a1b585b84e94
I20261120 03:00:49.610332  1189 consensus_queue.cc:261] T 06c3bac919e342b09dd6af13aebc387b P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.611320  1185 ts_tablet_manager.cc:1996] T 5328cf9c683a44548038a1b585b84e94 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
W20261120 03:00:49.611357   498 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet a72d5d7896de4020a1ac4d2f8c551e15 consensus state has no leader
I20261120 03:00:49.612612  1189 ts_tablet_manager.cc:1480] T 06c3bac919e342b09dd6af13aebc387b P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.042s	user 0.004s	sys 0.000s
I20261120 03:00:49.613827  1189 tablet_bootstrap.cc:492] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:49.614004   494 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 9f73b223756845bb8af7b7fd96c0f9fc (table default.Table_97c478725540413b876cdcaad37d3f1d [id=e5d7a83f0b454c539f39545c800f99ee]). Expected version 1 got 0
I20261120 03:00:49.615520   494 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 5f5c8711a6eb43c3aff1c27ed5115d6a (table default.Table_0a4ee63ac2c442c5bae84e199b8a9664 [id=affba127653b4dcbac7fc9fe76ba797a]). Expected version 2 got 0
I20261120 03:00:49.619196  1185 tablet_replica.cc:333] T 06c3bac919e342b09dd6af13aebc387b P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:49.619881  1185 raft_consensus.cc:2440] T 06c3bac919e342b09dd6af13aebc387b P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:49.620285  1185 raft_consensus.cc:2469] T 06c3bac919e342b09dd6af13aebc387b P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:49.622284  1180 tablet_bootstrap.cc:654] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.622880  1181 tablet_bootstrap.cc:654] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.628193  1189 tablet_bootstrap.cc:654] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.626932  1185 ts_tablet_manager.cc:1962] T 06c3bac919e342b09dd6af13aebc387b P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:49.646981  1189 tablet_bootstrap.cc:492] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:49.647374  1189 ts_tablet_manager.cc:1449] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.034s	user 0.009s	sys 0.000s
I20261120 03:00:49.652709  1180 tablet_bootstrap.cc:492] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:49.653118  1180 ts_tablet_manager.cc:1449] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.056s	user 0.006s	sys 0.002s
W20261120 03:00:49.655191   486 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.126:44467, user_credentials={real_user=slave}} blocked reactor thread for 62309.1us
I20261120 03:00:49.655560  1180 raft_consensus.cc:366] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.656177  1180 raft_consensus.cc:392] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.656432  1180 raft_consensus.cc:787] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.658336  1180 consensus_queue.cc:261] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.650038  1189 raft_consensus.cc:366] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.663481  1189 raft_consensus.cc:392] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.663717  1181 tablet_bootstrap.cc:492] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:49.663750  1189 raft_consensus.cc:787] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.664134  1181 ts_tablet_manager.cc:1449] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.057s	user 0.009s	sys 0.000s
I20261120 03:00:49.666284   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 5328cf9c683a44548038a1b585b84e94 (table default.Table_5e08feb4f2304718b4cc6c71a638e14c [id=59c24564b2eb44ccab8736e0ef5f733b]) successfully deleted
I20261120 03:00:49.666450  1181 raft_consensus.cc:366] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.667054  1181 raft_consensus.cc:392] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.667289  1181 raft_consensus.cc:787] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.670322  1180 ts_tablet_manager.cc:1480] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.017s	user 0.007s	sys 0.001s
I20261120 03:00:49.674322  1181 consensus_queue.cc:261] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.675675  1189 consensus_queue.cc:261] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.676517  1180 tablet_bootstrap.cc:492] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:49.681712  1185 ts_tablet_manager.cc:1975] T 06c3bac919e342b09dd6af13aebc387b P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:49.682165  1185 log.cc:1199] T 06c3bac919e342b09dd6af13aebc387b P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/06c3bac919e342b09dd6af13aebc387b
I20261120 03:00:49.682925   497 catalog_manager.cc:6146] Not sending DeleteTablet requests; no consensus state for tablet e843ce84994c4291a1cb4a9b6f94f4f6
I20261120 03:00:49.684700  1189 ts_tablet_manager.cc:1480] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.037s	user 0.002s	sys 0.004s
I20261120 03:00:49.682760  1185 ts_tablet_manager.cc:1996] T 06c3bac919e342b09dd6af13aebc387b P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:49.687014  1189 tablet_bootstrap.cc:492] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:49.691291   495 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_8c3f3e4b96ba410fa0109e46c25d4144"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:49.693488   495 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_8c3f3e4b96ba410fa0109e46c25d4144 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:49.696165   498 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_d1737969bdd644a6ab39da121eeedd60"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:49.697791   498 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d1737969bdd644a6ab39da121eeedd60 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:49.702419  1185 tablet_replica.cc:333] T a642c32c1b6043978516d97efa2347a3 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:49.703118  1185 raft_consensus.cc:2440] T a642c32c1b6043978516d97efa2347a3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:49.703543  1185 raft_consensus.cc:2469] T a642c32c1b6043978516d97efa2347a3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:49.704559  1189 tablet_bootstrap.cc:654] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.707756  1180 tablet_bootstrap.cc:654] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.711683  1185 ts_tablet_manager.cc:1962] T a642c32c1b6043978516d97efa2347a3 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:49.716750  1181 ts_tablet_manager.cc:1480] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.052s	user 0.007s	sys 0.000s
I20261120 03:00:49.717598  1181 tablet_bootstrap.cc:492] T 1237b0d2fa414ef38271835b5d69d6b9 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:49.720642  1189 tablet_bootstrap.cc:492] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:49.721037  1189 ts_tablet_manager.cc:1449] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.034s	user 0.006s	sys 0.006s
I20261120 03:00:49.723315  1189 raft_consensus.cc:366] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.729135  1181 tablet_bootstrap.cc:654] T 1237b0d2fa414ef38271835b5d69d6b9 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.723924  1189 raft_consensus.cc:392] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.743884  1189 raft_consensus.cc:787] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.744974  1189 consensus_queue.cc:261] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.746062  1181 tablet_bootstrap.cc:492] T 1237b0d2fa414ef38271835b5d69d6b9 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:49.746474  1181 ts_tablet_manager.cc:1449] T 1237b0d2fa414ef38271835b5d69d6b9 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.029s	user 0.015s	sys 0.001s
I20261120 03:00:49.748412  1189 ts_tablet_manager.cc:1480] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.027s	user 0.006s	sys 0.001s
I20261120 03:00:49.748684  1181 raft_consensus.cc:366] T 1237b0d2fa414ef38271835b5d69d6b9 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.749265  1181 raft_consensus.cc:392] T 1237b0d2fa414ef38271835b5d69d6b9 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.749522  1181 raft_consensus.cc:787] T 1237b0d2fa414ef38271835b5d69d6b9 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.750545  1181 consensus_queue.cc:261] T 1237b0d2fa414ef38271835b5d69d6b9 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.759231  1181 ts_tablet_manager.cc:1480] T 1237b0d2fa414ef38271835b5d69d6b9 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.012s	user 0.004s	sys 0.000s
I20261120 03:00:49.760252  1185 ts_tablet_manager.cc:1975] T a642c32c1b6043978516d97efa2347a3 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:49.760599  1185 log.cc:1199] T a642c32c1b6043978516d97efa2347a3 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/a642c32c1b6043978516d97efa2347a3
I20261120 03:00:49.761185  1185 ts_tablet_manager.cc:1996] T a642c32c1b6043978516d97efa2347a3 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:49.762365  1221 raft_consensus.cc:534] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:49.762773  1221 raft_consensus.cc:556] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.764642  1221 leader_election.cc:290] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:49.766573  1181 tablet_bootstrap.cc:492] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:49.768111  1079 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3f65ba863447413cb122e728df3632fd" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:49.769915   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 06c3bac919e342b09dd6af13aebc387b (table default.Table_91d5519024c0442b9cbce1cf67444e6e [id=927890104a3a4819878b6ae3f13c0e71]) successfully deleted
I20261120 03:00:49.772408  1181 tablet_bootstrap.cc:654] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:49.779937   709 leader_election.cc:343] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:49.784472   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3f65ba863447413cb122e728df3632fd" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:00:49.785856   710 leader_election.cc:343] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:49.786325   710 leader_election.cc:304] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:49.787107  1221 raft_consensus.cc:2966] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:49.799835  1180 tablet_bootstrap.cc:492] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:49.800241  1180 ts_tablet_manager.cc:1449] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.129s	user 0.018s	sys 0.023s
W20261120 03:00:49.800660   500 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 9a31431495274c648625b75900b4ac47 has no consensus state
I20261120 03:00:49.802568  1180 raft_consensus.cc:366] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.803166  1180 raft_consensus.cc:392] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.803977  1189 tablet_bootstrap.cc:492] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
W20261120 03:00:49.805480   486 consensus_peers.cc:597] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 -> Peer 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133): Couldn't send request to peer 2599fd9eef7d4ce990c40e2356a37c3f. Status: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20261120 03:00:49.807354  1263 tablet.cc:1872] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20261120 03:00:49.809825  1189 tablet_bootstrap.cc:654] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.803426  1180 raft_consensus.cc:787] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.813151  1180 consensus_queue.cc:261] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.814172  1262 raft_consensus.cc:534] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:49.814634  1262 raft_consensus.cc:556] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.816685  1262 leader_election.cc:290] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:49.817168  1221 raft_consensus.cc:534] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:49.817602  1221 raft_consensus.cc:556] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.819453  1221 leader_election.cc:290] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:49.820434   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5f5c8711a6eb43c3aff1c27ed5115d6a" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:49.821144   924 raft_consensus.cc:2062] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:49.821458   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5f5c8711a6eb43c3aff1c27ed5115d6a" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:49.822129   923 raft_consensus.cc:2062] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:00:49.822464   710 leader_election.cc:304] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:49.823752   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5f5c8711a6eb43c3aff1c27ed5115d6a" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:49.824115  1012 leader_election.cc:304] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:00:49.824438   777 raft_consensus.cc:2062] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:00:49.824878  1221 raft_consensus.cc:3024] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:49.825181  1221 raft_consensus.cc:534] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:49.825477  1221 raft_consensus.cc:3296] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:49.826960  1262 raft_consensus.cc:3024] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:49.827270  1262 raft_consensus.cc:534] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:49.827546  1262 raft_consensus.cc:3296] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:49.827744  1180 ts_tablet_manager.cc:1480] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.027s	user 0.005s	sys 0.005s
I20261120 03:00:49.828572  1180 tablet_bootstrap.cc:492] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:49.833223  1221 raft_consensus.cc:556] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.835886  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5f5c8711a6eb43c3aff1c27ed5115d6a" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:49.836585   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 53116.1us
I20261120 03:00:49.840844  1189 tablet_bootstrap.cc:492] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:49.841274  1189 ts_tablet_manager.cc:1449] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.037s	user 0.008s	sys 0.004s
I20261120 03:00:49.843740  1189 raft_consensus.cc:366] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.844378  1189 raft_consensus.cc:392] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.844648  1189 raft_consensus.cc:787] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.854718  1180 tablet_bootstrap.cc:654] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.857921  1189 consensus_queue.cc:261] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.865496  1221 leader_election.cc:290] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:49.866027  1221 raft_consensus.cc:534] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:49.866312   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5f5c8711a6eb43c3aff1c27ed5115d6a" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:49.866451  1221 raft_consensus.cc:556] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.866842   923 raft_consensus.cc:3296] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:49.867684   923 raft_consensus.cc:2062] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:49.868219  1221 leader_election.cc:290] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:49.871148  1189 ts_tablet_manager.cc:1480] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.030s	user 0.008s	sys 0.000s
I20261120 03:00:49.871006   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1237b0d2fa414ef38271835b5d69d6b9" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:00:49.873142   710 leader_election.cc:343] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:49.873911  1180 tablet_bootstrap.cc:492] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:49.874323  1180 ts_tablet_manager.cc:1449] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.046s	user 0.009s	sys 0.002s
I20261120 03:00:49.877154  1180 raft_consensus.cc:366] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.877764  1180 raft_consensus.cc:392] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:49.878005  1180 raft_consensus.cc:787] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:49.883941   710 leader_election.cc:304] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:49.884930  1077 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5f5c8711a6eb43c3aff1c27ed5115d6a" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:49.886237  1076 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1237b0d2fa414ef38271835b5d69d6b9" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:49.886955  1076 raft_consensus.cc:2062] T 1237b0d2fa414ef38271835b5d69d6b9 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:49.828894   497 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_d5a78163a78d417985bfaf9768756bd2"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:49.829811   502 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_b901331d0fdd40a89231797be95dd176" } new_table_name: "default.Table_4755bb4a11ae4e33bff2a6cc7384921a" modify_external_catalogs: true
I20261120 03:00:49.830582   500 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_f6f4dff944d24549b23b98158a9c0d4e"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:49.895292   500 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_f6f4dff944d24549b23b98158a9c0d4e in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:49.889288   493 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_3bb077af5e15449b85500aca3a1500c3" } modify_external_catalogs: true
I20261120 03:00:49.898272   493 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_3bb077af5e15449b85500aca3a1500c3" } modify_external_catalogs: true
I20261120 03:00:49.899232   709 leader_election.cc:304] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440
I20261120 03:00:49.900465  1288 raft_consensus.cc:3024] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:49.900812  1288 raft_consensus.cc:534] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
W20261120 03:00:49.900882   497 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d5a78163a78d417985bfaf9768756bd2 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:49.901146  1288 raft_consensus.cc:3296] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:49.901881  1288 raft_consensus.cc:556] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.910210  1180 consensus_queue.cc:261] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:49.912190  1180 ts_tablet_manager.cc:1480] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.038s	user 0.005s	sys 0.000s
I20261120 03:00:49.913046  1180 tablet_bootstrap.cc:492] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:49.914510  1283 raft_consensus.cc:3024] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:49.915680  1283 raft_consensus.cc:744] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:49.916527  1283 consensus_queue.cc:238] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.920317  1288 leader_election.cc:290] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:49.921434   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1237b0d2fa414ef38271835b5d69d6b9" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
W20261120 03:00:49.922832   710 leader_election.cc:343] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:49.922844  1079 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1237b0d2fa414ef38271835b5d69d6b9" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:49.923519  1079 raft_consensus.cc:3296] T 1237b0d2fa414ef38271835b5d69d6b9 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:49.931838  1079 raft_consensus.cc:2062] T 1237b0d2fa414ef38271835b5d69d6b9 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:49.937836  1286 tablet.cc:1872] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
W20261120 03:00:49.925879   486 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.124:38133, user_credentials={real_user=slave}} blocked reactor thread for 120435us
I20261120 03:00:49.958725   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet a642c32c1b6043978516d97efa2347a3 (table default.Table_e617c39b20b24b8f9477f625da46b661 [id=2b97828a5a874a8cb13e283ef679c7fb]) successfully deleted
I20261120 03:00:49.958712  1292 tablet.cc:1872] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
W20261120 03:00:49.959578   486 catalog_manager.cc:5048] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 136056e5242a42ef8fa4b40c989b5423 because the tablet was not found. No further retry: Not found: Tablet not found: 136056e5242a42ef8fa4b40c989b5423
I20261120 03:00:49.968072  1180 tablet_bootstrap.cc:654] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:49.968114  1262 raft_consensus.cc:556] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:49.969298  1078 raft_consensus.cc:2628] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5beeefe8bef54caeac463d05d8c30fa6 in current term 1: Already voted for candidate 823abdbd90ba47bbacad0f8dcff78234 in this term.
I20261120 03:00:49.974298  1262 leader_election.cc:290] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:49.975806   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5f5c8711a6eb43c3aff1c27ed5115d6a" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:49.977726   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5f5c8711a6eb43c3aff1c27ed5115d6a" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:49.978475   923 raft_consensus.cc:2628] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 823abdbd90ba47bbacad0f8dcff78234 in current term 1: Already voted for candidate 5beeefe8bef54caeac463d05d8c30fa6 in this term.
I20261120 03:00:49.979702  1012 leader_election.cc:304] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:49.981524  1289 consensus_queue.cc:1049] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20261120 03:00:49.982869  1289 consensus_queue.cc:1049] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:49.998986  1290 raft_consensus.cc:2966] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20261120 03:00:50.004307   496 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 9f73b223756845bb8af7b7fd96c0f9fc
W20261120 03:00:50.009191   501 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.65:40295 (request call id 23) took 1000 ms (client timeout 1000 ms). Trace:
I20261120 03:00:50.024546   498 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_34bfe641479b42ceac031dafd7e21d2d" } new_table_name: "default.Table_650dc67e0ac74c149a3f64d456767938" modify_external_catalogs: true
W20261120 03:00:50.009570   501 rpcz_store.cc:269] 1120 03:00:49.008580 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:49.068212 (+ 59632us) service_pool.cc:224] Handling call
1120 03:00:49.230322 (+162110us) spinlock_profiling.cc:245] Waited 161 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07d9ce 00007f155abf0174 00007f155abf0102 00007f155abf0031 00007f155abeffac 00007f155abef262 00007f1558e990bc 00007f1558e984f6
1120 03:00:50.009122 (+778800us) inbound_call.cc:177] Queueing success response
Metrics: {"reported_tablets":3,"spinlock_wait_cycles":369437440}
I20261120 03:00:50.057786  1290 raft_consensus.cc:534] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:50.058235  1290 raft_consensus.cc:556] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.058538   902 tablet_service.cc:1542] Processing DeleteTablet for tablet 9f73b223756845bb8af7b7fd96c0f9fc with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:50.060190  1290 leader_election.cc:290] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:50.061658   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a642c32c1b6043978516d97efa2347a3" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:50.062460   924 raft_consensus.cc:2062] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
W20261120 03:00:50.062980  1011 leader_election.cc:343] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: a642c32c1b6043978516d97efa2347a3
I20261120 03:00:50.064252  1012 leader_election.cc:304] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:50.065301  1290 raft_consensus.cc:3024] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:50.065599  1290 raft_consensus.cc:534] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:50.065877  1290 raft_consensus.cc:3296] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:50.066566  1290 raft_consensus.cc:556] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.078639  1290 leader_election.cc:290] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:50.080353   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a642c32c1b6043978516d97efa2347a3" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:50.080937   924 raft_consensus.cc:3296] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:50.081820   924 raft_consensus.cc:2062] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
W20261120 03:00:50.079286  1011 leader_election.cc:343] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: a642c32c1b6043978516d97efa2347a3
I20261120 03:00:50.099249  1012 leader_election.cc:304] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:50.103035  1290 raft_consensus.cc:3024] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:50.140688  1262 raft_consensus.cc:534] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:50.141287   385 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
/tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/wal
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/logs
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.124:38133
--webserver_interface=127.0.96.124
--webserver_port=43247
--builtin_ntp_servers=127.0.96.84:35145
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.96.126:44467,127.0.96.125:37411,127.0.96.124:38133
--hive_metastore_notification_log_poll_period_seconds=1
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
I20261120 03:00:50.141273  1262 raft_consensus.cc:556] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.151445   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d753dd170cab4f18980e85707c4093e9" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:50.152213   923 raft_consensus.cc:2062] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:00:50.152103   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d753dd170cab4f18980e85707c4093e9" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:50.152801   777 raft_consensus.cc:2062] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:00:50.153781  1012 leader_election.cc:304] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:00:50.156003  1262 leader_election.cc:290] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:50.156507  1262 raft_consensus.cc:3024] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:50.156800  1262 raft_consensus.cc:534] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:50.157100  1262 raft_consensus.cc:3296] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:50.157781  1262 raft_consensus.cc:556] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.160859  1295 tablet_replica.cc:333] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:50.161676  1295 raft_consensus.cc:2440] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:50.162490  1290 raft_consensus.cc:744] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Becoming Leader. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Running, Role: LEADER
I20261120 03:00:50.163520  1290 consensus_queue.cc:238] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:50.171558   486 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.126:44467, user_credentials={real_user=slave}} blocked reactor thread for 174523us
I20261120 03:00:50.183588  1290 consensus_queue.cc:1049] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:50.189719  1058 tablet_service.cc:1542] Processing DeleteTablet for tablet 9f73b223756845bb8af7b7fd96c0f9fc with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:50.194897  1303 tablet_replica.cc:333] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:50.195736  1303 raft_consensus.cc:2440] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:50.198356  1297 raft_consensus.cc:534] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:50.199183  1295 raft_consensus.cc:2469] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:50.204735  1295 ts_tablet_manager.cc:1962] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:50.199012  1303 raft_consensus.cc:2469] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:50.206355  1297 raft_consensus.cc:556] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:50.217331  1181 tablet_bootstrap.cc:492] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:50.217782  1181 ts_tablet_manager.cc:1449] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.451s	user 0.040s	sys 0.087s
I20261120 03:00:50.220064  1181 raft_consensus.cc:366] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.220715  1181 raft_consensus.cc:392] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:50.220952  1181 raft_consensus.cc:787] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:50.221879   501 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_9e8b5f30c7164ece9a605f3187e9e777"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:50.223646   501 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_9e8b5f30c7164ece9a605f3187e9e777 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:50.231635  1280 raft_consensus.cc:534] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:50.232151  1280 raft_consensus.cc:556] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.236804   709 leader_election.cc:304] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440
I20261120 03:00:50.239667  1079 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "06c3bac919e342b09dd6af13aebc387b" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:50.240175  1280 leader_election.cc:290] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:50.243156   755 tablet_service.cc:1542] Processing DeleteTablet for tablet 9f73b223756845bb8af7b7fd96c0f9fc with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:50.244542   757 tablet_service.cc:1542] Processing DeleteTablet for tablet 5328cf9c683a44548038a1b585b84e94 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:47 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:50.245393   497 master_service.cc:965] ReplaceTablet: received request to replace tablet a72d5d7896de4020a1ac4d2f8c551e15 from {username='slave'} at 127.0.0.1:45640
W20261120 03:00:50.246412  1011 consensus_peers.cc:597] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 -> Peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Couldn't send request to peer 5beeefe8bef54caeac463d05d8c30fa6. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a642c32c1b6043978516d97efa2347a3. This is attempt 1: this message will repeat every 5th retry.
W20261120 03:00:50.245020   857 leader_election.cc:343] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: 06c3bac919e342b09dd6af13aebc387b
I20261120 03:00:50.251358  1185 tablet_replica.cc:333] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:50.252293  1262 leader_election.cc:290] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:50.252974  1185 raft_consensus.cc:2440] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:50.253117  1289 raft_consensus.cc:3024] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:50.253937  1289 raft_consensus.cc:744] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:50.254292  1185 raft_consensus.cc:2469] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:50.257251  1289 consensus_queue.cc:238] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.258088   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d753dd170cab4f18980e85707c4093e9" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:50.258966   777 raft_consensus.cc:3296] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:50.260094   777 raft_consensus.cc:2062] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
I20261120 03:00:50.258636  1185 ts_tablet_manager.cc:1962] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:50.260648   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "c3b56d0ccab140c28a37808d9d5902c6" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:50.261368   924 raft_consensus.cc:2062] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:00:50.269622  1012 leader_election.cc:304] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:00:50.271289  1262 raft_consensus.cc:3024] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:50.271672  1262 raft_consensus.cc:534] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:50.271978  1262 raft_consensus.cc:3296] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:50.272751  1262 raft_consensus.cc:556] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:50.277087  1283 raft_consensus.cc:534] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:50.277597  1283 raft_consensus.cc:556] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.279737  1283 leader_election.cc:290] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:50.281551  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a72d5d7896de4020a1ac4d2f8c551e15" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:50.282325  1078 raft_consensus.cc:2062] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:50.283111   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a72d5d7896de4020a1ac4d2f8c551e15" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:50.283910   924 raft_consensus.cc:2062] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:50.283799   709 leader_election.cc:304] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:00:50.284677  1283 raft_consensus.cc:3024] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:50.285004  1283 raft_consensus.cc:534] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:50.285156  1262 leader_election.cc:290] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:50.285315  1283 raft_consensus.cc:3296] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:50.286873   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "c3b56d0ccab140c28a37808d9d5902c6" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
W20261120 03:00:50.288460  1011 leader_election.cc:343] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: c3b56d0ccab140c28a37808d9d5902c6
I20261120 03:00:50.290321   924 raft_consensus.cc:3296] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
W20261120 03:00:50.290671  1011 leader_election.cc:343] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: c3b56d0ccab140c28a37808d9d5902c6
I20261120 03:00:50.291253   924 raft_consensus.cc:2062] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
I20261120 03:00:50.221544  1181 consensus_queue.cc:261] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.292999  1079 raft_consensus.cc:2062] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:50.293345  1297 leader_election.cc:290] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:50.293738  1283 raft_consensus.cc:556] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.296471  1079 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a72d5d7896de4020a1ac4d2f8c551e15" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:50.297008  1079 raft_consensus.cc:3296] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:50.297916  1079 raft_consensus.cc:2062] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:50.298111  1180 tablet_bootstrap.cc:492] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:50.298514  1180 ts_tablet_manager.cc:1449] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.386s	user 0.009s	sys 0.008s
I20261120 03:00:50.298910  1283 leader_election.cc:290] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:50.296963   500 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_f6f4dff944d24549b23b98158a9c0d4e" } modify_external_catalogs: true
I20261120 03:00:50.300161   500 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_f6f4dff944d24549b23b98158a9c0d4e" } modify_external_catalogs: true
I20261120 03:00:50.300881  1180 raft_consensus.cc:366] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.301448  1180 raft_consensus.cc:392] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:50.301692  1180 raft_consensus.cc:787] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:50.296219   496 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_d2f8156a63404ca5a92c6ea7190d15ad"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:50.305014   496 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d2f8156a63404ca5a92c6ea7190d15ad in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:50.307837   922 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a72d5d7896de4020a1ac4d2f8c551e15" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:50.308425   922 raft_consensus.cc:3296] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:50.309351   922 raft_consensus.cc:2062] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:50.310673  1181 ts_tablet_manager.cc:1480] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.093s	user 0.006s	sys 0.001s
I20261120 03:00:50.318866  1181 tablet_bootstrap.cc:492] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:50.325614   710 leader_election.cc:304] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:50.326447  1283 raft_consensus.cc:3024] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:50.326902  1283 raft_consensus.cc:744] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:50.328114  1283 consensus_queue.cc:238] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.329697   857 leader_election.cc:304] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:50.333751  1282 raft_consensus.cc:3024] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:50.334256  1282 raft_consensus.cc:534] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:50.334650  1282 raft_consensus.cc:3296] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:50.338394  1180 consensus_queue.cc:261] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.344385  1181 tablet_bootstrap.cc:654] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:50.344964  1282 raft_consensus.cc:556] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.340670  1185 ts_tablet_manager.cc:1975] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20261120 03:00:50.348750  1185 log.cc:1199] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/9f73b223756845bb8af7b7fd96c0f9fc
I20261120 03:00:50.349707  1185 ts_tablet_manager.cc:1996] T 9f73b223756845bb8af7b7fd96c0f9fc P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:50.351058  1303 ts_tablet_manager.cc:1962] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:50.353089   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 9f73b223756845bb8af7b7fd96c0f9fc (table default.Table_97c478725540413b876cdcaad37d3f1d [id=e5d7a83f0b454c539f39545c800f99ee]) successfully deleted
W20261120 03:00:50.353871   486 catalog_manager.cc:5048] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 5328cf9c683a44548038a1b585b84e94 because the tablet was not found. No further retry: Not found: Tablet not found: 5328cf9c683a44548038a1b585b84e94
I20261120 03:00:50.356009  1295 ts_tablet_manager.cc:1975] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20261120 03:00:50.295117  1011 leader_election.cc:304] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
W20261120 03:00:50.358734  1011 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 63690.2us
I20261120 03:00:50.358868  1221 consensus_queue.cc:1049] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:50.360082  1221 consensus_queue.cc:1049] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:50.360507  1181 tablet_bootstrap.cc:492] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:50.365478  1181 ts_tablet_manager.cc:1449] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.047s	user 0.009s	sys 0.001s
I20261120 03:00:50.366335  1295 log.cc:1199] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/9f73b223756845bb8af7b7fd96c0f9fc
I20261120 03:00:50.369143  1181 raft_consensus.cc:366] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.371129  1295 ts_tablet_manager.cc:1996] T 9f73b223756845bb8af7b7fd96c0f9fc P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
W20261120 03:00:50.376246   710 consensus_peers.cc:597] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Couldn't send request to peer 3d30c2428730481db9b0ffd279dd3440. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:50.377641  1288 consensus_queue.cc:1049] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20261120 03:00:50.379922   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d753dd170cab4f18980e85707c4093e9" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:50.373829  1296 raft_consensus.cc:3024] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:50.381362  1296 raft_consensus.cc:744] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Becoming Leader. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Running, Role: LEADER
I20261120 03:00:50.382114   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 9f73b223756845bb8af7b7fd96c0f9fc (table default.Table_97c478725540413b876cdcaad37d3f1d [id=e5d7a83f0b454c539f39545c800f99ee]) successfully deleted
I20261120 03:00:50.383875   923 raft_consensus.cc:3296] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:50.382278  1296 consensus_queue.cc:238] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.387902  1012 leader_election.cc:304] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:50.389178  1290 raft_consensus.cc:3024] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:50.389752  1181 raft_consensus.cc:392] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:50.390061  1181 raft_consensus.cc:787] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:50.389899  1290 raft_consensus.cc:744] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Becoming Leader. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Running, Role: LEADER
W20261120 03:00:50.385015   857 leader_election.cc:343] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: 06c3bac919e342b09dd6af13aebc387b
I20261120 03:00:50.393123   923 raft_consensus.cc:2062] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
I20261120 03:00:50.397086  1290 consensus_queue.cc:238] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:50.390684  1282 leader_election.cc:290] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:50.407431  1301 consensus_queue.cc:1049] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20261120 03:00:50.424749  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "06c3bac919e342b09dd6af13aebc387b" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:50.437700  1078 raft_consensus.cc:3296] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:50.438712  1078 raft_consensus.cc:2062] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:00:50.391800  1181 consensus_queue.cc:261] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.446439  1180 ts_tablet_manager.cc:1480] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.148s	user 0.014s	sys 0.005s
I20261120 03:00:50.447382  1180 tablet_bootstrap.cc:492] T 1237b0d2fa414ef38271835b5d69d6b9 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:50.451998   500 catalog_manager.cc:6146] Not sending DeleteTablet requests; no consensus state for tablet 1454d724715748e38978b7ae4e884623
I20261120 03:00:50.452987   500 master_service.cc:965] ReplaceTablet: received request to replace tablet 1cf6d449fa2845a7b1f4cbd1de639d20 from {username='slave'} at 127.0.0.1:45640
I20261120 03:00:50.456933   501 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_5303db2882934be1bfc2251ce51f2c8d"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:50.457677  1180 tablet_bootstrap.cc:654] T 1237b0d2fa414ef38271835b5d69d6b9 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:50.468453  1181 ts_tablet_manager.cc:1480] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.102s	user 0.006s	sys 0.000s
I20261120 03:00:50.476104  1058 tablet_service.cc:1495] Processing CreateTablet for tablet df61317717d3401eb74cd93aa336c4dc (DEFAULT_TABLE table=default.Table_5203afc2eeba4e2abc5a94484597a25a [id=b30488f6d14b4c9ab1566f2376be602a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.477273  1058 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet df61317717d3401eb74cd93aa336c4dc. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:50.476953   757 tablet_service.cc:1495] Processing CreateTablet for tablet df61317717d3401eb74cd93aa336c4dc (DEFAULT_TABLE table=default.Table_5203afc2eeba4e2abc5a94484597a25a [id=b30488f6d14b4c9ab1566f2376be602a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.477521  1056 tablet_service.cc:1495] Processing CreateTablet for tablet 9a31431495274c648625b75900b4ac47 (DEFAULT_TABLE table=default.Table_3bb077af5e15449b85500aca3a1500c3 [id=234b77d1cf1847b0ae8afa8d3dbdff71]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.478137   757 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet df61317717d3401eb74cd93aa336c4dc. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:50.478907  1056 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9a31431495274c648625b75900b4ac47. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:50.484771   902 tablet_service.cc:1495] Processing CreateTablet for tablet df61317717d3401eb74cd93aa336c4dc (DEFAULT_TABLE table=default.Table_5203afc2eeba4e2abc5a94484597a25a [id=b30488f6d14b4c9ab1566f2376be602a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.485867   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet df61317717d3401eb74cd93aa336c4dc. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:50.489856   901 tablet_service.cc:1495] Processing CreateTablet for tablet 9a31431495274c648625b75900b4ac47 (DEFAULT_TABLE table=default.Table_3bb077af5e15449b85500aca3a1500c3 [id=234b77d1cf1847b0ae8afa8d3dbdff71]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.491505   901 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9a31431495274c648625b75900b4ac47. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:00:50.467566   488 catalog_manager.cc:4863] Async tablet task AlterTable RPC for tablet 9a31431495274c648625b75900b4ac47 (table default.Table_3bb077af5e15449b85500aca3a1500c3 [id=234b77d1cf1847b0ae8afa8d3dbdff71], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 9a31431495274c648625b75900b4ac47 consensus state has no leader
W20261120 03:00:50.494921   501 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_5303db2882934be1bfc2251ce51f2c8d in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:50.469527   495 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet a72d5d7896de4020a1ac4d2f8c551e15 (table default.Table_f601bffe6ddb49b3a3c589ac7b25235c [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]). Expected version 1 got 0
I20261120 03:00:50.502616   495 catalog_manager.cc:5856] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65). New cstate: current_term: 1 leader_uuid: "5beeefe8bef54caeac463d05d8c30fa6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } }
I20261120 03:00:50.503976   495 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 5f5c8711a6eb43c3aff1c27ed5115d6a (table default.Table_0a4ee63ac2c442c5bae84e199b8a9664 [id=affba127653b4dcbac7fc9fe76ba797a]). Expected version 2 got 0
I20261120 03:00:50.512688   493 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 9a31431495274c648625b75900b4ac47
I20261120 03:00:50.513051   904 tablet_service.cc:1495] Processing CreateTablet for tablet 0fd4a37075a34ec6bb0d6eb950045a1b (DEFAULT_TABLE table=default.Table_d52207672f8f40acb7026fec3eb73d85 [id=563f250386d546eb8b53db93e8139e9b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.514158   904 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0fd4a37075a34ec6bb0d6eb950045a1b. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:50.513895   755 tablet_service.cc:1495] Processing CreateTablet for tablet 9a31431495274c648625b75900b4ac47 (DEFAULT_TABLE table=default.Table_3bb077af5e15449b85500aca3a1500c3 [id=234b77d1cf1847b0ae8afa8d3dbdff71]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.514993   755 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9a31431495274c648625b75900b4ac47. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:50.519443   903 tablet_service.cc:1542] Processing DeleteTablet for tablet 9a31431495274c648625b75900b4ac47 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:44896
W20261120 03:00:50.523347   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 9a31431495274c648625b75900b4ac47 because the tablet was not found. No further retry: Not found: Tablet not found: 9a31431495274c648625b75900b4ac47
I20261120 03:00:50.525187  1059 tablet_service.cc:1542] Processing DeleteTablet for tablet 9a31431495274c648625b75900b4ac47 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:50.527624   754 tablet_service.cc:1542] Processing DeleteTablet for tablet 9a31431495274c648625b75900b4ac47 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:50.527904  1296 consensus_queue.cc:1049] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20261120 03:00:50.529145   486 catalog_manager.cc:5048] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 9a31431495274c648625b75900b4ac47 because the tablet was not found. No further retry: Not found: Tablet not found: 9a31431495274c648625b75900b4ac47
I20261120 03:00:50.530663   857 leader_election.cc:304] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:50.531816  1311 raft_consensus.cc:3024] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:50.532241  1311 raft_consensus.cc:744] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:00:50.491564   496 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_85ad53c3650a452b9c9ab5d515add49c"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:50.535693  1181 tablet_bootstrap.cc:492] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:50.537118  1059 tablet_service.cc:1495] Processing CreateTablet for tablet 0fd4a37075a34ec6bb0d6eb950045a1b (DEFAULT_TABLE table=default.Table_d52207672f8f40acb7026fec3eb73d85 [id=563f250386d546eb8b53db93e8139e9b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.538106  1059 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0fd4a37075a34ec6bb0d6eb950045a1b. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:50.539645   754 tablet_service.cc:1495] Processing CreateTablet for tablet 0fd4a37075a34ec6bb0d6eb950045a1b (DEFAULT_TABLE table=default.Table_d52207672f8f40acb7026fec3eb73d85 [id=563f250386d546eb8b53db93e8139e9b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.540664   754 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0fd4a37075a34ec6bb0d6eb950045a1b. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:50.532977  1311 consensus_queue.cc:238] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:50.546480   496 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_85ad53c3650a452b9c9ab5d515add49c in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:50.581130  1284 consensus_queue.cc:1049] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
W20261120 03:00:50.584820  1011 consensus_peers.cc:597] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 -> Peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Couldn't send request to peer 5beeefe8bef54caeac463d05d8c30fa6. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c3b56d0ccab140c28a37808d9d5902c6. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:50.598270  1181 tablet_bootstrap.cc:654] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:50.614630  1303 ts_tablet_manager.cc:1975] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20261120 03:00:50.616241  1303 log.cc:1199] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/9f73b223756845bb8af7b7fd96c0f9fc
I20261120 03:00:50.621969  1303 ts_tablet_manager.cc:1996] T 9f73b223756845bb8af7b7fd96c0f9fc P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:50.623422   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 9f73b223756845bb8af7b7fd96c0f9fc (table default.Table_97c478725540413b876cdcaad37d3f1d [id=e5d7a83f0b454c539f39545c800f99ee]) successfully deleted
I20261120 03:00:50.630456   902 tablet_service.cc:1495] Processing CreateTablet for tablet fa57d0c282964e37b9133ea57f458a51 (DEFAULT_TABLE table=default.Table_dd2f7617ff0e401b9b862d5991110dd5 [id=3955f41b6d4d41bf8ba2c81e4ac7f4a8]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.632519  1056 tablet_service.cc:1495] Processing CreateTablet for tablet fa57d0c282964e37b9133ea57f458a51 (DEFAULT_TABLE table=default.Table_dd2f7617ff0e401b9b862d5991110dd5 [id=3955f41b6d4d41bf8ba2c81e4ac7f4a8]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.632679   753 tablet_service.cc:1495] Processing CreateTablet for tablet fa57d0c282964e37b9133ea57f458a51 (DEFAULT_TABLE table=default.Table_dd2f7617ff0e401b9b862d5991110dd5 [id=3955f41b6d4d41bf8ba2c81e4ac7f4a8]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.633678  1181 tablet_bootstrap.cc:492] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:50.633797   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fa57d0c282964e37b9133ea57f458a51. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:50.633848  1056 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fa57d0c282964e37b9133ea57f458a51. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:50.634087  1181 ts_tablet_manager.cc:1449] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.165s	user 0.006s	sys 0.005s
I20261120 03:00:50.636795  1181 raft_consensus.cc:366] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.637396  1181 raft_consensus.cc:392] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:50.637650  1181 raft_consensus.cc:787] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:50.638288  1181 consensus_queue.cc:261] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.640187  1181 ts_tablet_manager.cc:1480] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.006s	user 0.004s	sys 0.000s
I20261120 03:00:50.640993  1181 tablet_bootstrap.cc:492] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
W20261120 03:00:50.643298   486 catalog_manager.cc:5061] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 9a31431495274c648625b75900b4ac47 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 9a31431495274c648625b75900b4ac47 already in progress: creating tablet
I20261120 03:00:50.650512   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fa57d0c282964e37b9133ea57f458a51. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:50.659986   493 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_9e8b5f30c7164ece9a605f3187e9e777" } modify_external_catalogs: true
I20261120 03:00:50.660565   493 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_9e8b5f30c7164ece9a605f3187e9e777" } modify_external_catalogs: true
I20261120 03:00:50.676169  1283 raft_consensus.cc:534] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:50.676631  1283 raft_consensus.cc:556] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.679888   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "136056e5242a42ef8fa4b40c989b5423" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:00:50.681530   710 leader_election.cc:343] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:50.683152  1283 leader_election.cc:290] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:50.688128  1079 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "136056e5242a42ef8fa4b40c989b5423" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:50.688863  1079 raft_consensus.cc:2062] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:50.691933   709 leader_election.cc:304] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440
I20261120 03:00:50.692632   500 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 1cf6d449fa2845a7b1f4cbd1de639d20
I20261120 03:00:50.695313   901 tablet_service.cc:1542] Processing DeleteTablet for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 with delete_type TABLET_DATA_DELETED (replaced by tablet 68179488b9bc4e03b65a9f1d434c95b5) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:50.696069  1295 tablet_replica.cc:333] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:50.696823   500 catalog_manager.cc:6952] ReplaceTablet: tablet 1cf6d449fa2845a7b1f4cbd1de639d20 deleted and replaced by tablet 68179488b9bc4e03b65a9f1d434c95b5
I20261120 03:00:50.696873  1295 raft_consensus.cc:2440] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:50.697407  1295 raft_consensus.cc:2469] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:50.698340  1283 raft_consensus.cc:3024] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:50.698691  1283 raft_consensus.cc:534] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:50.698993  1283 raft_consensus.cc:3296] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:50.699505  1295 ts_tablet_manager.cc:1962] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:50.699757  1283 raft_consensus.cc:556] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.708987  1283 leader_election.cc:290] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:50.710178   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "136056e5242a42ef8fa4b40c989b5423" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:50.710660  1079 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "136056e5242a42ef8fa4b40c989b5423" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:50.710994   756 tablet_service.cc:1542] Processing DeleteTablet for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 with delete_type TABLET_DATA_DELETED (replaced by tablet 68179488b9bc4e03b65a9f1d434c95b5) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:50.711242  1079 raft_consensus.cc:3296] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:50.711717  1185 tablet_replica.cc:333] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:50.712420  1185 raft_consensus.cc:2440] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:50.712908  1185 raft_consensus.cc:2469] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:50.714418  1181 tablet_bootstrap.cc:654] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:50.715020  1185 ts_tablet_manager.cc:1962] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
W20261120 03:00:50.711598   710 leader_election.cc:343] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:50.717935  1079 raft_consensus.cc:2062] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:50.720685   709 leader_election.cc:304] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440
I20261120 03:00:50.725490  1283 raft_consensus.cc:3024] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:50.725665  1180 tablet_bootstrap.cc:492] T 1237b0d2fa414ef38271835b5d69d6b9 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:50.726095  1180 ts_tablet_manager.cc:1449] T 1237b0d2fa414ef38271835b5d69d6b9 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.279s	user 0.008s	sys 0.017s
I20261120 03:00:50.723121  1324 tablet_bootstrap.cc:492] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:50.728315  1056 tablet_service.cc:1542] Processing DeleteTablet for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 with delete_type TABLET_DATA_DELETED (replaced by tablet 68179488b9bc4e03b65a9f1d434c95b5) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:50.728431  1180 raft_consensus.cc:366] T 1237b0d2fa414ef38271835b5d69d6b9 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.729017  1303 tablet_replica.cc:333] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:50.729022  1180 raft_consensus.cc:392] T 1237b0d2fa414ef38271835b5d69d6b9 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:50.729326  1180 raft_consensus.cc:787] T 1237b0d2fa414ef38271835b5d69d6b9 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:50.729836  1303 raft_consensus.cc:2440] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:50.730830  1303 raft_consensus.cc:2469] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:50.699440   500 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_3ce972a9130f459f940b2910e9eb1427"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:50.733116  1303 ts_tablet_manager.cc:1962] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
W20261120 03:00:50.733263   500 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_3ce972a9130f459f940b2910e9eb1427 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:00:50.738777   857 consensus_peers.cc:597] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 -> Peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Couldn't send request to peer 5beeefe8bef54caeac463d05d8c30fa6. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 06c3bac919e342b09dd6af13aebc387b. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:50.729947  1180 consensus_queue.cc:261] T 1237b0d2fa414ef38271835b5d69d6b9 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.745577   901 tablet_service.cc:1495] Processing CreateTablet for tablet 5fee8dff291c43e9be0fb10b308718ce (DEFAULT_TABLE table=default.Table_a1f866996679431e9e1c4c5f6f8cdd12 [id=a275e215155b4824a1494f0954577fe8]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.747282   901 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5fee8dff291c43e9be0fb10b308718ce. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:50.756147  1056 tablet_service.cc:1495] Processing CreateTablet for tablet 5fee8dff291c43e9be0fb10b308718ce (DEFAULT_TABLE table=default.Table_a1f866996679431e9e1c4c5f6f8cdd12 [id=a275e215155b4824a1494f0954577fe8]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.757310  1056 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5fee8dff291c43e9be0fb10b308718ce. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:50.726045  1283 raft_consensus.cc:744] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:50.759169  1280 consensus_queue.cc:1049] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:50.759480  1283 consensus_queue.cc:238] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.764748   904 tablet_service.cc:1495] Processing CreateTablet for tablet 4abb67f3819a45b9b05dde04842e2c9d (DEFAULT_TABLE table=default.Table_69539e86f82e4df2a4b6888aa55293fc [id=0335c1c0df984b0ab347ae9e6d566c5d]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.765939   904 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4abb67f3819a45b9b05dde04842e2c9d. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:50.779373   756 tablet_service.cc:1495] Processing CreateTablet for tablet 4abb67f3819a45b9b05dde04842e2c9d (DEFAULT_TABLE table=default.Table_69539e86f82e4df2a4b6888aa55293fc [id=0335c1c0df984b0ab347ae9e6d566c5d]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.780522   756 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4abb67f3819a45b9b05dde04842e2c9d. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:50.786907   753 tablet_service.cc:1495] Processing CreateTablet for tablet 5fee8dff291c43e9be0fb10b308718ce (DEFAULT_TABLE table=default.Table_a1f866996679431e9e1c4c5f6f8cdd12 [id=a275e215155b4824a1494f0954577fe8]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.788117   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5fee8dff291c43e9be0fb10b308718ce. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:50.794906  1326 tablet.cc:1872] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20261120 03:00:50.796947  1180 ts_tablet_manager.cc:1480] T 1237b0d2fa414ef38271835b5d69d6b9 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.071s	user 0.006s	sys 0.000s
I20261120 03:00:50.808354  1295 ts_tablet_manager.cc:1975] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20261120 03:00:50.816972  1324 tablet_bootstrap.cc:654] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:50.824397   710 consensus_peers.cc:597] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Couldn't send request to peer 3d30c2428730481db9b0ffd279dd3440. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:50.834306  1295 log.cc:1199] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/1cf6d449fa2845a7b1f4cbd1de639d20
I20261120 03:00:50.835246  1295 ts_tablet_manager.cc:1996] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:50.828305  1327 tablet.cc:1872] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20261120 03:00:50.840339  1185 ts_tablet_manager.cc:1975] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20261120 03:00:50.817699  1180 tablet_bootstrap.cc:492] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:50.840726  1185 log.cc:1199] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/1cf6d449fa2845a7b1f4cbd1de639d20
I20261120 03:00:50.841388  1185 ts_tablet_manager.cc:1996] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:50.846807  1303 ts_tablet_manager.cc:1975] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20261120 03:00:50.856307  1180 tablet_bootstrap.cc:654] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:50.856690  1303 log.cc:1199] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/1cf6d449fa2845a7b1f4cbd1de639d20
I20261120 03:00:50.863627   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (table default.Table_34bfe641479b42ceac031dafd7e21d2d [id=cfcd5fea97fb4ab3baff99eb28cd9476]) successfully deleted
I20261120 03:00:50.867259  1303 ts_tablet_manager.cc:1996] T 1cf6d449fa2845a7b1f4cbd1de639d20 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:50.870280  1324 tablet_bootstrap.cc:492] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:50.870836  1324 ts_tablet_manager.cc:1449] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.148s	user 0.002s	sys 0.011s
I20261120 03:00:50.874444  1324 raft_consensus.cc:366] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:50.877684  1324 raft_consensus.cc:392] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:50.878036  1324 raft_consensus.cc:787] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:50.882423   497 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet a72d5d7896de4020a1ac4d2f8c551e15
I20261120 03:00:50.884011   497 catalog_manager.cc:6952] ReplaceTablet: tablet a72d5d7896de4020a1ac4d2f8c551e15 deleted and replaced by tablet ce612f9a121b4b18908724d2726e156f
I20261120 03:00:50.884949   497 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_51385081065c4b11955d65846ec63d6f"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:50.887681   499 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 5f5c8711a6eb43c3aff1c27ed5115d6a (table default.Table_0a4ee63ac2c442c5bae84e199b8a9664 [id=affba127653b4dcbac7fc9fe76ba797a]). Expected version 2 got 0
W20261120 03:00:50.890156   502 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 3f65ba863447413cb122e728df3632fd consensus state has no leader
I20261120 03:00:50.890623   502 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:45640 (request call id 68) took 1062 ms. Trace:
I20261120 03:00:50.892436   757 tablet_service.cc:1542] Processing DeleteTablet for tablet a72d5d7896de4020a1ac4d2f8c551e15 with delete_type TABLET_DATA_DELETED (replaced by tablet ce612f9a121b4b18908724d2726e156f) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:50.890895   502 rpcz_store.cc:276] 1120 03:00:49.828369 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:49.829531 (+  1162us) service_pool.cc:224] Handling call
1120 03:00:49.891737 (+ 62206us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:00:49.892396 (+   659us) catalog_manager.cc:3769] Apply alter schema
1120 03:00:49.892940 (+   544us) catalog_manager.cc:3801] Acquired catalog manager lock
1120 03:00:49.893148 (+   208us) catalog_manager.cc:3972] Updating metadata on disk
1120 03:00:50.234852 (+341704us) catalog_manager.cc:4003] Committing alterations to in-memory state
1120 03:00:50.890565 (+655713us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:49.908784 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:49.909170 (+   386us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:49.909192 (+    22us) write_op.cc:440] Acquired schema lock
1120 03:00:49.909205 (+    13us) tablet.cc:710] Decoding operations
1120 03:00:49.909440 (+   235us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:49.909473 (+    33us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:49.909494 (+    21us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:00:49.909624 (+   130us) tablet.cc:758] Row locks acquired
1120 03:00:49.909635 (+    11us) write_op.cc:260] PREPARE: finished
1120 03:00:49.909733 (+    98us) write_op.cc:270] Start()
1120 03:00:49.909802 (+    69us) write_op.cc:276] Timestamp: P: 1795143649909696 usec, L: 0
1120 03:00:49.909816 (+    14us) op_driver.cc:350] REPLICATION: starting
1120 03:00:49.910529 (+   713us) log.cc:844] Serialized 404 byte log entry
1120 03:00:50.222640 (+312111us) op_driver.cc:466] REPLICATION: finished
1120 03:00:50.225361 (+  2721us) write_op.cc:301] APPLY: starting
1120 03:00:50.225406 (+    45us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:50.225496 (+    90us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:50.225587 (+    91us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:50.225721 (+   134us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:50.225846 (+   125us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:50.225887 (+    41us) write_op.cc:312] APPLY: finished
1120 03:00:50.226048 (+   161us) log.cc:844] Serialized 22 byte log entry
1120 03:00:50.234483 (+  8435us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:50.234551 (+    68us) write_op.cc:459] Released schema lock
1120 03:00:50.234581 (+    30us) write_op.cc:346] FINISH: Updating metrics
1120 03:00:50.234837 (+   256us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:50.234864 (+    27us) write_op.cc:459] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":2588,"apply.run_cpu_time_us":1243,"apply.run_wall_time_us":9438,"num_ops":1,"prepare.queue_time_us":92,"prepare.run_cpu_time_us":1674,"prepare.run_wall_time_us":2948,"replication_time_us":312749}]]}
I20261120 03:00:50.894268   494 catalog_manager.cc:6409] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: default.Table_34bfe641479b42ceac031dafd7e21d2d [id=cfcd5fea97fb4ab3baff99eb28cd9476] alter complete (version 2)
I20261120 03:00:50.894222  1324 consensus_queue.cc:261] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
W20261120 03:00:50.896931   494 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.67:53803 (request call id 13) took 1505 ms (client timeout 1000 ms). Trace:
I20261120 03:00:50.903744  1180 tablet_bootstrap.cc:492] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:50.904189  1180 ts_tablet_manager.cc:1449] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.087s	user 0.005s	sys 0.006s
I20261120 03:00:50.906576  1180 raft_consensus.cc:366] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:50.912621   487 catalog_manager.cc:5154] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): alter failed for tablet 9f73b223756845bb8af7b7fd96c0f9fc (table default.Table_97c478725540413b876cdcaad37d3f1d [id=e5d7a83f0b454c539f39545c800f99ee]),no further retry: Not found: Tablet not found: 9f73b223756845bb8af7b7fd96c0f9fc
I20261120 03:00:50.911091   904 tablet_service.cc:1542] Processing DeleteTablet for tablet a72d5d7896de4020a1ac4d2f8c551e15 with delete_type TABLET_DATA_DELETED (replaced by tablet ce612f9a121b4b18908724d2726e156f) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:50.913599  1180 raft_consensus.cc:392] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:50.913887  1180 raft_consensus.cc:787] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:50.914037  1295 tablet_replica.cc:333] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:50.914925  1295 raft_consensus.cc:2440] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:50.915429  1295 raft_consensus.cc:2469] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:50.915630  1324 ts_tablet_manager.cc:1480] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.044s	user 0.007s	sys 0.000s
I20261120 03:00:50.914557  1180 consensus_queue.cc:261] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:50.917937  1180 ts_tablet_manager.cc:1480] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.013s	user 0.007s	sys 0.000s
I20261120 03:00:50.916501  1324 tablet_bootstrap.cc:492] T df61317717d3401eb74cd93aa336c4dc P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:50.918907  1295 ts_tablet_manager.cc:1962] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:50.921007  1180 tablet_bootstrap.cc:492] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
W20261120 03:00:50.923379   495 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.66:36141 (request call id 19) took 828 ms (client timeout 1000 ms). Trace:
I20261120 03:00:50.924821   496 master_service.cc:965] ReplaceTablet: received request to replace tablet d753dd170cab4f18980e85707c4093e9 from {username='slave'} at 127.0.0.1:45640
I20261120 03:00:50.929229   904 tablet_service.cc:1542] Processing DeleteTablet for tablet 9f73b223756845bb8af7b7fd96c0f9fc with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:50.929888   902 tablet_service.cc:1542] Processing DeleteTablet for tablet a642c32c1b6043978516d97efa2347a3 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:50.930550   903 tablet_service.cc:1542] Processing DeleteTablet for tablet 06c3bac919e342b09dd6af13aebc387b with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:44896
W20261120 03:00:50.897290   494 rpcz_store.cc:269] 1120 03:00:49.391954 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:49.393552 (+  1598us) service_pool.cc:224] Handling call
1120 03:00:50.896847 (+1503295us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:50.054381 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:50.054768 (+   387us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:50.054792 (+    24us) write_op.cc:440] Acquired schema lock
1120 03:00:50.054806 (+    14us) tablet.cc:710] Decoding operations
1120 03:00:50.055056 (+   250us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:50.055101 (+    45us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:50.055123 (+    22us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:00:50.055267 (+   144us) tablet.cc:758] Row locks acquired
1120 03:00:50.055278 (+    11us) write_op.cc:260] PREPARE: finished
1120 03:00:50.055386 (+   108us) write_op.cc:270] Start()
1120 03:00:50.055462 (+    76us) write_op.cc:276] Timestamp: P: 1795143650055332 usec, L: 0
1120 03:00:50.055477 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:00:50.055927 (+   450us) log.cc:844] Serialized 355 byte log entry
1120 03:00:50.223071 (+167144us) op_driver.cc:466] REPLICATION: finished
1120 03:00:50.227847 (+  4776us) write_op.cc:301] APPLY: starting
1120 03:00:50.227897 (+    50us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:50.227979 (+    82us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:50.228074 (+    95us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:50.228197 (+   123us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:50.228305 (+   108us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:50.228348 (+    43us) write_op.cc:312] APPLY: finished
1120 03:00:50.228499 (+   151us) log.cc:844] Serialized 22 byte log entry
1120 03:00:50.228619 (+   120us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:50.228684 (+    65us) write_op.cc:459] Released schema lock
1120 03:00:50.228716 (+    32us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"reported_tablets":3,"child_traces":[["op",{"apply.queue_time_us":4658,"apply.run_cpu_time_us":1041,"apply.run_wall_time_us":1041,"num_ops":1,"prepare.queue_time_us":41810,"prepare.run_cpu_time_us":1820,"prepare.run_wall_time_us":2948,"raft.queue_time_us":98,"raft.run_cpu_time_us":12,"raft.run_wall_time_us":13,"replication_time_us":167523,"thread_start_us":71240,"threads_started":1}]]}
I20261120 03:00:50.931136   900 tablet_service.cc:1542] Processing DeleteTablet for tablet c3b56d0ccab140c28a37808d9d5902c6 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:50.893764   502 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_d5a78163a78d417985bfaf9768756bd2" } modify_external_catalogs: true
I20261120 03:00:50.932394   899 tablet_service.cc:1542] Processing DeleteTablet for tablet d753dd170cab4f18980e85707c4093e9 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:50.932312   502 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_d5a78163a78d417985bfaf9768756bd2" } modify_external_catalogs: true
I20261120 03:00:50.934878  1180 tablet_bootstrap.cc:654] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:50.935802   497 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_51385081065c4b11955d65846ec63d6f in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:00:50.923738   495 rpcz_store.cc:269] 1120 03:00:50.094758 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:50.094898 (+   140us) service_pool.cc:224] Handling call
1120 03:00:50.923311 (+828413us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:50.570447 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:50.570843 (+   396us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:50.570865 (+    22us) write_op.cc:440] Acquired schema lock
1120 03:00:50.570877 (+    12us) tablet.cc:710] Decoding operations
1120 03:00:50.571136 (+   259us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:50.571176 (+    40us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:50.571196 (+    20us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:00:50.571355 (+   159us) tablet.cc:758] Row locks acquired
1120 03:00:50.571367 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:00:50.571480 (+   113us) write_op.cc:270] Start()
1120 03:00:50.571558 (+    78us) write_op.cc:276] Timestamp: P: 1795143650571420 usec, L: 0
1120 03:00:50.571573 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:00:50.572054 (+   481us) log.cc:844] Serialized 526 byte log entry
1120 03:00:50.586203 (+ 14149us) op_driver.cc:466] REPLICATION: finished
1120 03:00:50.586604 (+   401us) write_op.cc:301] APPLY: starting
1120 03:00:50.586661 (+    57us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:50.586755 (+    94us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:50.587386 (+   631us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:50.587564 (+   178us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:50.587697 (+   133us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:50.587739 (+    42us) write_op.cc:312] APPLY: finished
1120 03:00:50.587917 (+   178us) log.cc:844] Serialized 22 byte log entry
1120 03:00:50.590004 (+  2087us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:50.590087 (+    83us) write_op.cc:459] Released schema lock
1120 03:00:50.590123 (+    36us) write_op.cc:346] FINISH: Updating metrics
1120 03:00:50.646099 (+ 55976us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:50.646124 (+    25us) write_op.cc:459] Released schema lock
Metrics: {"reported_tablets":8,"child_traces":[["op",{"apply.queue_time_us":247,"apply.run_cpu_time_us":1785,"apply.run_wall_time_us":3715,"num_ops":1,"prepare.queue_time_us":64467,"prepare.run_cpu_time_us":2034,"prepare.run_wall_time_us":75621,"raft.queue_time_us":165,"raft.run_cpu_time_us":1168,"raft.run_wall_time_us":1167,"replication_time_us":14534}]]}
I20261120 03:00:50.951236  1324 tablet_bootstrap.cc:654] T df61317717d3401eb74cd93aa336c4dc P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:50.962321   754 tablet_service.cc:1495] Processing CreateTablet for tablet a1f6197c60dd4dfb8aeb5c38373aab21 (DEFAULT_TABLE table=default.Table_0b2a37ca199f431fb72c9bb9db84eef3 [id=6a353839c63741c3a49ec2e5724283f5]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.963572   754 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a1f6197c60dd4dfb8aeb5c38373aab21. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:50.965553   495 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 3f65ba863447413cb122e728df3632fd (table default.Table_4755bb4a11ae4e33bff2a6cc7384921a [id=676ae8fc1a8c488d820a11b930b0ed5f]). Expected version 1 got 0
I20261120 03:00:50.968884  1295 ts_tablet_manager.cc:1975] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:50.969379  1295 log.cc:1199] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/a72d5d7896de4020a1ac4d2f8c551e15
I20261120 03:00:50.970217  1295 ts_tablet_manager.cc:1996] T a72d5d7896de4020a1ac4d2f8c551e15 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:50.973587  1295 tablet_replica.cc:333] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:50.973618   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet a72d5d7896de4020a1ac4d2f8c551e15 (table default.Table_f601bffe6ddb49b3a3c589ac7b25235c [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]) successfully deleted
W20261120 03:00:50.974471   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 9f73b223756845bb8af7b7fd96c0f9fc because the tablet was not found. No further retry: Not found: Tablet not found: 9f73b223756845bb8af7b7fd96c0f9fc
I20261120 03:00:50.981329  1295 raft_consensus.cc:2440] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:50.981962  1295 raft_consensus.cc:2469] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:50.985589   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (table default.Table_34bfe641479b42ceac031dafd7e21d2d [id=cfcd5fea97fb4ab3baff99eb28cd9476]) successfully deleted
I20261120 03:00:50.990146  1295 ts_tablet_manager.cc:1962] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:50.996878   493 catalog_manager.cc:6146] Not sending DeleteTablet requests; no consensus state for tablet 11a71c10fa994b789e593004de182084
I20261120 03:00:50.994496   899 tablet_service.cc:1495] Processing CreateTablet for tablet 5a3fe1499d0b4fab9f9949c8ded0a8dc (DEFAULT_TABLE table=default.Table_14745a85367e453183d6a0a197be3fbb [id=c27784e6e0c84353a8aec497dd69b9ff]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:50.998886   899 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5a3fe1499d0b4fab9f9949c8ded0a8dc. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:51.008288  1180 tablet_bootstrap.cc:492] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:51.008766  1180 ts_tablet_manager.cc:1449] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.089s	user 0.021s	sys 0.037s
I20261120 03:00:51.009285  1324 tablet_bootstrap.cc:492] T df61317717d3401eb74cd93aa336c4dc P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:51.009704  1324 ts_tablet_manager.cc:1449] T df61317717d3401eb74cd93aa336c4dc P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.093s	user 0.010s	sys 0.000s
I20261120 03:00:51.011587  1180 raft_consensus.cc:366] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:50.982530   901 tablet_service.cc:1495] Processing CreateTablet for tablet a1f6197c60dd4dfb8aeb5c38373aab21 (DEFAULT_TABLE table=default.Table_0b2a37ca199f431fb72c9bb9db84eef3 [id=6a353839c63741c3a49ec2e5724283f5]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:51.012221  1180 raft_consensus.cc:392] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:51.012507  1180 raft_consensus.cc:787] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:51.013204  1324 raft_consensus.cc:366] T df61317717d3401eb74cd93aa336c4dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.013851  1324 raft_consensus.cc:392] T df61317717d3401eb74cd93aa336c4dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:51.014137  1324 raft_consensus.cc:787] T df61317717d3401eb74cd93aa336c4dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:51.006906  1185 tablet_replica.cc:333] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:51.014843  1324 consensus_queue.cc:261] T df61317717d3401eb74cd93aa336c4dc P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.015632  1185 raft_consensus.cc:2440] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:51.020860   901 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a1f6197c60dd4dfb8aeb5c38373aab21. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:51.023119   756 tablet_service.cc:1495] Processing CreateTablet for tablet 5a3fe1499d0b4fab9f9949c8ded0a8dc (DEFAULT_TABLE table=default.Table_14745a85367e453183d6a0a197be3fbb [id=c27784e6e0c84353a8aec497dd69b9ff]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:51.028491   755 tablet_service.cc:1495] Processing CreateTablet for tablet 00dd2ffad1124244b171b2657d38a708 (DEFAULT_TABLE table=default.Table_e07b2ba4d9864d148d7088622b10e03c [id=201bf626c7d643d7a98a680b1e714ba3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:51.029462   900 tablet_service.cc:1495] Processing CreateTablet for tablet 00dd2ffad1124244b171b2657d38a708 (DEFAULT_TABLE table=default.Table_e07b2ba4d9864d148d7088622b10e03c [id=201bf626c7d643d7a98a680b1e714ba3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:51.031288   900 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00dd2ffad1124244b171b2657d38a708. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:51.034590   756 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5a3fe1499d0b4fab9f9949c8ded0a8dc. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:51.035974   755 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00dd2ffad1124244b171b2657d38a708. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:51.042336  1185 raft_consensus.cc:2469] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:51.046342  1180 consensus_queue.cc:261] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.047323  1324 ts_tablet_manager.cc:1480] T df61317717d3401eb74cd93aa336c4dc P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.037s	user 0.000s	sys 0.009s
I20261120 03:00:51.048271  1324 tablet_bootstrap.cc:492] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:51.053018  1185 ts_tablet_manager.cc:1962] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:51.054533   493 catalog_manager.cc:5856] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65). New cstate: current_term: 1 leader_uuid: "5beeefe8bef54caeac463d05d8c30fa6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: UNKNOWN } } }
I20261120 03:00:51.059582   493 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 3f65ba863447413cb122e728df3632fd (table default.Table_4755bb4a11ae4e33bff2a6cc7384921a [id=676ae8fc1a8c488d820a11b930b0ed5f]). Expected version 1 got 0
I20261120 03:00:51.053088  1295 ts_tablet_manager.cc:1975] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:51.063107  1295 log.cc:1199] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/a642c32c1b6043978516d97efa2347a3
I20261120 03:00:51.063946  1295 ts_tablet_manager.cc:1996] T a642c32c1b6043978516d97efa2347a3 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:51.064679  1311 raft_consensus.cc:534] T 1237b0d2fa414ef38271835b5d69d6b9 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:51.065479  1311 raft_consensus.cc:556] T 1237b0d2fa414ef38271835b5d69d6b9 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.069211   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1237b0d2fa414ef38271835b5d69d6b9" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:51.066988  1180 ts_tablet_manager.cc:1480] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.058s	user 0.002s	sys 0.005s
I20261120 03:00:51.078956   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet a642c32c1b6043978516d97efa2347a3 (table default.Table_e617c39b20b24b8f9477f625da46b661 [id=2b97828a5a874a8cb13e283ef679c7fb]) successfully deleted
I20261120 03:00:51.067479  1311 leader_election.cc:290] T 1237b0d2fa414ef38271835b5d69d6b9 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:51.079694  1180 tablet_bootstrap.cc:492] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:51.081388  1295 tablet_replica.cc:333] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:51.082667  1295 raft_consensus.cc:2440] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:51.089715  1295 pending_rounds.cc:70] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440: Trying to abort 1 pending ops.
I20261120 03:00:51.094228  1295 pending_rounds.cc:77] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7352908392629579776 op_type: NO_OP noop_request { }
I20261120 03:00:51.094805  1295 raft_consensus.cc:3115] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
I20261120 03:00:51.095794  1295 raft_consensus.cc:2469] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:51.107749  1295 ts_tablet_manager.cc:1962] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:51.113464  1311 raft_consensus.cc:534] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:51.114471  1311 raft_consensus.cc:556] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.116681  1311 leader_election.cc:290] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:51.118481   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3f65ba863447413cb122e728df3632fd" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:51.119186   777 raft_consensus.cc:2062] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:51.120379   857 leader_election.cc:304] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:51.114022  1180 tablet_bootstrap.cc:654] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:51.122514   501 master_service.cc:965] ReplaceTablet: received request to replace tablet d753dd170cab4f18980e85707c4093e9 from {username='slave'} at 127.0.0.1:45640
I20261120 03:00:51.122956  1311 raft_consensus.cc:3024] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:51.123276  1311 raft_consensus.cc:534] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:51.123577  1311 raft_consensus.cc:3296] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:51.124356  1311 raft_consensus.cc:556] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.133350  1311 leader_election.cc:290] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:51.133920  1324 tablet_bootstrap.cc:654] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:51.135288   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3f65ba863447413cb122e728df3632fd" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:51.135843   777 raft_consensus.cc:3296] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:51.141368  1185 ts_tablet_manager.cc:1975] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:51.141523   496 master_service.cc:965] ReplaceTablet: received request to replace tablet df61317717d3401eb74cd93aa336c4dc from {username='slave'} at 127.0.0.1:45640
I20261120 03:00:51.141709  1185 log.cc:1199] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/a72d5d7896de4020a1ac4d2f8c551e15
I20261120 03:00:51.142246   777 raft_consensus.cc:2062] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:00:51.142398  1185 ts_tablet_manager.cc:1996] T a72d5d7896de4020a1ac4d2f8c551e15 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:51.145332   857 leader_election.cc:304] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:51.149435  1311 raft_consensus.cc:3024] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:51.151233   494 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_3c79e29085ad4875874397eaefcab250"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:51.153129   494 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_3c79e29085ad4875874397eaefcab250 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:00:51.159471   498 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 68179488b9bc4e03b65a9f1d434c95b5 has no consensus state
I20261120 03:00:51.160012   498 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:45640 (request call id 70) took 1144 ms. Trace:
I20261120 03:00:51.164904  1311 raft_consensus.cc:744] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:00:51.165866  1311 consensus_queue.cc:238] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.169030  1324 tablet_bootstrap.cc:492] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:51.169519  1324 ts_tablet_manager.cc:1449] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.121s	user 0.006s	sys 0.006s
I20261120 03:00:51.170526   500 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_5203afc2eeba4e2abc5a94484597a25a" } new_table_name: "default.Table_237971f3240b4730a6b7716296374a46" modify_external_catalogs: true
I20261120 03:00:51.173095  1324 raft_consensus.cc:366] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.173818  1324 raft_consensus.cc:392] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:51.176322  1180 tablet_bootstrap.cc:492] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:51.176733  1180 ts_tablet_manager.cc:1449] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.097s	user 0.027s	sys 0.010s
I20261120 03:00:51.178144   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet a72d5d7896de4020a1ac4d2f8c551e15 (table default.Table_f601bffe6ddb49b3a3c589ac7b25235c [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]) successfully deleted
I20261120 03:00:51.179512  1180 raft_consensus.cc:366] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.180488  1180 raft_consensus.cc:392] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:51.180745  1180 raft_consensus.cc:787] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:51.181370  1180 consensus_queue.cc:261] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.182209   502 catalog_manager.cc:6146] Not sending DeleteTablet requests; no consensus state for tablet 867cbfa03a5a43c99172f9a608713441
I20261120 03:00:51.184034  1180 ts_tablet_manager.cc:1480] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.007s	user 0.007s	sys 0.000s
W20261120 03:00:51.184580   499 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.65:40295 (request call id 24) took 1162 ms (client timeout 1000 ms). Trace:
I20261120 03:00:51.184815  1180 tablet_bootstrap.cc:492] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
W20261120 03:00:51.185042   499 rpcz_store.cc:269] 1120 03:00:50.022472 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:50.026307 (+  3835us) service_pool.cc:224] Handling call
1120 03:00:50.041634 (+ 15327us) spinlock_profiling.cc:245] Waited 1.76 ms on lock 0x7b54000501a8. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155d7d7791 00007f155d7d6d6c 00007f155cf7fce4 00007f155d07d9ce 00007f155abf0174 00007f155abf0102 00007f155abf0031 00007f155abeffac 00007f155abef262 00007f1558e990bc 00007f1558e984f6
1120 03:00:51.184505 (+1142871us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":27208,"reported_tablets":8,"spinlock_wait_cycles":4039424}
I20261120 03:00:51.160305   498 rpcz_store.cc:276] 1120 03:00:50.015553 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:50.024187 (+  8634us) service_pool.cc:224] Handling call
1120 03:00:50.025175 (+   988us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:00:50.902222 (+877047us) catalog_manager.cc:3769] Apply alter schema
1120 03:00:50.903047 (+   825us) catalog_manager.cc:3801] Acquired catalog manager lock
1120 03:00:50.903326 (+   279us) catalog_manager.cc:3972] Updating metadata on disk
1120 03:00:51.158878 (+255552us) catalog_manager.cc:4003] Committing alterations to in-memory state
1120 03:00:51.159956 (+  1078us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:51.015385 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:51.015789 (+   404us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:51.015814 (+    25us) write_op.cc:440] Acquired schema lock
1120 03:00:51.015827 (+    13us) tablet.cc:710] Decoding operations
1120 03:00:51.016088 (+   261us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:51.016122 (+    34us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:51.016142 (+    20us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:00:51.016295 (+   153us) tablet.cc:758] Row locks acquired
1120 03:00:51.016307 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:00:51.062470 (+ 46163us) write_op.cc:270] Start()
1120 03:00:51.062570 (+   100us) write_op.cc:276] Timestamp: P: 1795143651030156 usec, L: 0
1120 03:00:51.062585 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:00:51.063070 (+   485us) log.cc:844] Serialized 404 byte log entry
1120 03:00:51.155048 (+ 91978us) op_driver.cc:466] REPLICATION: finished
1120 03:00:51.156330 (+  1282us) write_op.cc:301] APPLY: starting
1120 03:00:51.156390 (+    60us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:51.156485 (+    95us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:51.156596 (+   111us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:51.156748 (+   152us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:51.156866 (+   118us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:51.156906 (+    40us) write_op.cc:312] APPLY: finished
1120 03:00:51.157097 (+   191us) log.cc:844] Serialized 22 byte log entry
1120 03:00:51.157259 (+   162us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:51.157319 (+    60us) write_op.cc:459] Released schema lock
1120 03:00:51.157351 (+    32us) write_op.cc:346] FINISH: Updating metrics
1120 03:00:51.157590 (+   239us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:51.157612 (+    22us) write_op.cc:459] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":1141,"apply.run_cpu_time_us":1199,"apply.run_wall_time_us":1199,"num_ops":1,"prepare.queue_time_us":110649,"prepare.run_cpu_time_us":2061,"prepare.run_wall_time_us":58868,"raft.queue_time_us":3727,"raft.run_cpu_time_us":15,"raft.run_wall_time_us":15,"replication_time_us":92351}]]}
I20261120 03:00:51.194993  1180 tablet_bootstrap.cc:654] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:51.180052   755 tablet_service.cc:1542] Processing DeleteTablet for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 with delete_type TABLET_DATA_DELETED (replaced by tablet 68179488b9bc4e03b65a9f1d434c95b5) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:51.199052   753 tablet_service.cc:1542] Processing DeleteTablet for tablet 136056e5242a42ef8fa4b40c989b5423 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:51.200192  1185 tablet_replica.cc:333] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:51.200874  1185 raft_consensus.cc:2440] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:51.201021   753 tablet_service.cc:1542] Processing DeleteTablet for tablet d753dd170cab4f18980e85707c4093e9 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:51.201565  1185 pending_rounds.cc:70] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6: Trying to abort 1 pending ops.
W20261120 03:00:51.201723   486 catalog_manager.cc:5048] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 because the tablet was not found. No further retry: Not found: Tablet not found: 1cf6d449fa2845a7b1f4cbd1de639d20
I20261120 03:00:51.201517   499 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_4755bb4a11ae4e33bff2a6cc7384921a" } modify_external_catalogs: true
I20261120 03:00:51.201817  1185 pending_rounds.cc:77] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7352908393702359040 op_type: NO_OP noop_request { }
I20261120 03:00:51.202193  1185 raft_consensus.cc:3115] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
I20261120 03:00:51.202499  1185 raft_consensus.cc:2469] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:51.202332   499 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_4755bb4a11ae4e33bff2a6cc7384921a" } modify_external_catalogs: true
I20261120 03:00:51.203111   754 tablet_service.cc:1542] Processing DeleteTablet for tablet 9f73b223756845bb8af7b7fd96c0f9fc with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:51.182143  1324 raft_consensus.cc:787] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:51.204282  1324 consensus_queue.cc:261] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.204958  1185 ts_tablet_manager.cc:1962] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:51.206394  1324 ts_tablet_manager.cc:1480] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.037s	user 0.005s	sys 0.007s
I20261120 03:00:51.206943  1336 consensus_queue.cc:1049] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:51.209190  1324 tablet_bootstrap.cc:492] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:51.215878  1324 tablet_bootstrap.cc:654] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:51.216915  1283 raft_consensus.cc:534] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:51.217386  1283 raft_consensus.cc:556] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.218793  1180 tablet_bootstrap.cc:492] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:51.219213  1180 ts_tablet_manager.cc:1449] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.035s	user 0.009s	sys 0.000s
I20261120 03:00:51.219300  1283 leader_election.cc:290] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:51.221009   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0fd4a37075a34ec6bb0d6eb950045a1b" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:51.221810   923 raft_consensus.cc:2062] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:51.222409  1180 raft_consensus.cc:366] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.222988  1180 raft_consensus.cc:392] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:51.223229  1180 raft_consensus.cc:787] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:51.223327   710 leader_election.cc:304] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:51.223804  1180 consensus_queue.cc:261] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.226130  1283 raft_consensus.cc:3024] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:51.226480  1283 raft_consensus.cc:534] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:51.226857  1283 raft_consensus.cc:3296] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:51.228008  1180 ts_tablet_manager.cc:1480] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.009s	user 0.006s	sys 0.000s
I20261120 03:00:51.229667  1180 tablet_bootstrap.cc:492] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:51.232434  1283 raft_consensus.cc:556] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.239331  1180 tablet_bootstrap.cc:654] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:51.239387   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0fd4a37075a34ec6bb0d6eb950045a1b" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:51.239962   923 raft_consensus.cc:3296] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:51.241201   923 raft_consensus.cc:2062] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:51.244616   497 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_58a2bd95297c48189f4c9338a266cabd"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:51.246969   497 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_58a2bd95297c48189f4c9338a266cabd in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:51.249926   498 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_fd084b0b1e49435c8bbbceca8642a022"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:51.251629   498 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_fd084b0b1e49435c8bbbceca8642a022 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:51.258679  1283 leader_election.cc:290] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:51.264621   922 raft_consensus.cc:3296] T 1237b0d2fa414ef38271835b5d69d6b9 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:51.265744  1185 ts_tablet_manager.cc:1975] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:51.280176  1185 log.cc:1199] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/136056e5242a42ef8fa4b40c989b5423
I20261120 03:00:51.280972  1185 ts_tablet_manager.cc:1996] T 136056e5242a42ef8fa4b40c989b5423 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:51.293066   495 catalog_manager.cc:6409] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: default.Table_0a4ee63ac2c442c5bae84e199b8a9664 [id=affba127653b4dcbac7fc9fe76ba797a] alter complete (version 2)
I20261120 03:00:51.302870  1180 tablet_bootstrap.cc:492] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:51.303299  1180 ts_tablet_manager.cc:1449] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.074s	user 0.014s	sys 0.022s
I20261120 03:00:51.305708  1180 raft_consensus.cc:366] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.306362  1180 raft_consensus.cc:392] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:51.306609  1180 raft_consensus.cc:787] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:51.307219  1180 consensus_queue.cc:261] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.310886   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 136056e5242a42ef8fa4b40c989b5423 (table default.Table_d2ca6a35f8b842d784167fd22339644a [id=afefca6af8d64812b4f3e37317a1c8d7]) successfully deleted
I20261120 03:00:51.313267  1185 tablet_replica.cc:333] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:51.313560  1295 ts_tablet_manager.cc:1975] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:51.314018  1295 log.cc:1199] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/06c3bac919e342b09dd6af13aebc387b
I20261120 03:00:51.315071  1180 ts_tablet_manager.cc:1480] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.012s	user 0.005s	sys 0.001s
I20261120 03:00:51.315937  1180 tablet_bootstrap.cc:492] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:51.318315  1185 raft_consensus.cc:2440] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:51.323439  1185 raft_consensus.cc:2469] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:51.327579  1180 tablet_bootstrap.cc:654] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:51.331700  1185 ts_tablet_manager.cc:1962] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:51.333966   710 leader_election.cc:304] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:51.334911  1283 raft_consensus.cc:3024] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:51.335623  1283 raft_consensus.cc:744] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:51.336469  1283 consensus_queue.cc:238] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.346010  1180 tablet_bootstrap.cc:492] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:51.346453  1180 ts_tablet_manager.cc:1449] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.031s	user 0.012s	sys 0.002s
I20261120 03:00:51.353884  1180 raft_consensus.cc:366] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.354540  1180 raft_consensus.cc:392] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:51.354797  1180 raft_consensus.cc:787] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:51.356936  1180 consensus_queue.cc:261] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.360133  1180 ts_tablet_manager.cc:1480] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.013s	user 0.005s	sys 0.000s
I20261120 03:00:51.361001  1180 tablet_bootstrap.cc:492] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:51.362047  1289 consensus_queue.cc:1049] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20261120 03:00:51.371402  1180 tablet_bootstrap.cc:654] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:51.372903  1295 ts_tablet_manager.cc:1996] T 06c3bac919e342b09dd6af13aebc387b P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:51.376744  1324 tablet_bootstrap.cc:492] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:51.376735   496 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet df61317717d3401eb74cd93aa336c4dc
I20261120 03:00:51.377197  1324 ts_tablet_manager.cc:1449] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.168s	user 0.031s	sys 0.046s
I20261120 03:00:51.380031   496 catalog_manager.cc:6952] ReplaceTablet: tablet df61317717d3401eb74cd93aa336c4dc deleted and replaced by tablet 88b9e9692a874ae993dde4b9e2586a70
I20261120 03:00:51.380640  1324 raft_consensus.cc:366] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.381307  1324 raft_consensus.cc:392] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:51.381562  1324 raft_consensus.cc:787] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:51.382282  1324 consensus_queue.cc:261] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.382444  1283 raft_consensus.cc:534] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:51.383414   756 tablet_service.cc:1542] Processing DeleteTablet for tablet df61317717d3401eb74cd93aa336c4dc with delete_type TABLET_DATA_DELETED (replaced by tablet 88b9e9692a874ae993dde4b9e2586a70) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:51.383538  1283 raft_consensus.cc:556] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.386516  1283 leader_election.cc:290] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:51.387207  1324 ts_tablet_manager.cc:1480] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.010s	user 0.005s	sys 0.000s
I20261120 03:00:51.388562  1324 tablet_bootstrap.cc:492] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:51.388413   496 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_d52207672f8f40acb7026fec3eb73d85" } modify_external_catalogs: true
I20261120 03:00:51.389000   496 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_d52207672f8f40acb7026fec3eb73d85" } modify_external_catalogs: true
I20261120 03:00:51.393559   900 tablet_service.cc:1542] Processing DeleteTablet for tablet df61317717d3401eb74cd93aa336c4dc with delete_type TABLET_DATA_DELETED (replaced by tablet 88b9e9692a874ae993dde4b9e2586a70) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:51.395919  1324 tablet_bootstrap.cc:654] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:51.396391   922 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "fa57d0c282964e37b9133ea57f458a51" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:51.397104   922 raft_consensus.cc:2062] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:51.396894   501 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_af067a13691947e1a29b9461e9fe301e" } new_table_name: "default.Table_523caa128be441e1a52767e3d3318ce0" modify_external_catalogs: true
I20261120 03:00:51.398262   710 leader_election.cc:304] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:51.421867  1283 raft_consensus.cc:3024] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:51.422283  1283 raft_consensus.cc:534] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:51.422640  1283 raft_consensus.cc:3296] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:51.434440  1283 raft_consensus.cc:556] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.437119  1311 raft_consensus.cc:534] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:51.437568  1311 raft_consensus.cc:556] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.437871   922 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "fa57d0c282964e37b9133ea57f458a51" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:51.438417   922 raft_consensus.cc:3296] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:51.439462  1311 leader_election.cc:290] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:51.442747   922 raft_consensus.cc:2062] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:51.444411   494 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_54b560cd2410426782bbd02c54839db2"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:51.446717   494 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_54b560cd2410426782bbd02c54839db2 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:51.447163   710 leader_election.cc:304] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:51.448163  1289 raft_consensus.cc:3024] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:51.450757  1280 raft_consensus.cc:534] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:51.451189  1280 raft_consensus.cc:556] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.453801  1280 leader_election.cc:290] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:51.455955  1185 ts_tablet_manager.cc:1975] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:51.456359  1185 log.cc:1199] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/d753dd170cab4f18980e85707c4093e9
I20261120 03:00:51.457079  1185 ts_tablet_manager.cc:1996] T d753dd170cab4f18980e85707c4093e9 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:51.458841  1289 raft_consensus.cc:744] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:51.460377  1289 consensus_queue.cc:238] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.462464   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "4abb67f3819a45b9b05dde04842e2c9d" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
W20261120 03:00:51.465939   857 leader_election.cc:343] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: 136056e5242a42ef8fa4b40c989b5423
I20261120 03:00:51.469169   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 06c3bac919e342b09dd6af13aebc387b (table default.Table_91d5519024c0442b9cbce1cf67444e6e [id=927890104a3a4819878b6ae3f13c0e71]) successfully deleted
I20261120 03:00:51.470202  1295 tablet_replica.cc:333] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:51.471611  1295 raft_consensus.cc:2440] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:51.472162  1324 tablet_bootstrap.cc:492] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:51.472201  1295 raft_consensus.cc:2469] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:51.472584  1324 ts_tablet_manager.cc:1449] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.085s	user 0.034s	sys 0.033s
I20261120 03:00:51.474009   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet d753dd170cab4f18980e85707c4093e9 (table default.Table_269fbc3b47a44895be4b7bebddeee096 [id=62fab6f5923b4aa3be379f3d3d8d3046]) successfully deleted
I20261120 03:00:51.474848  1180 tablet_bootstrap.cc:492] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:51.475271  1180 ts_tablet_manager.cc:1449] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.114s	user 0.013s	sys 0.006s
I20261120 03:00:51.477622  1180 raft_consensus.cc:366] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.478287  1180 raft_consensus.cc:392] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:51.478538  1180 raft_consensus.cc:787] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
W20261120 03:00:51.480227   857 leader_election.cc:343] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:51.479281  1180 consensus_queue.cc:261] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.484303  1180 ts_tablet_manager.cc:1480] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.009s	user 0.007s	sys 0.000s
W20261120 03:00:51.485477   500 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 88b9e9692a874ae993dde4b9e2586a70 has no consensus state
I20261120 03:00:51.486889  1324 raft_consensus.cc:366] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.487519  1324 raft_consensus.cc:392] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:51.487795  1324 raft_consensus.cc:787] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:51.489449  1180 tablet_bootstrap.cc:492] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
W20261120 03:00:51.495149   486 catalog_manager.cc:5048] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 9f73b223756845bb8af7b7fd96c0f9fc because the tablet was not found. No further retry: Not found: Tablet not found: 9f73b223756845bb8af7b7fd96c0f9fc
I20261120 03:00:51.497208  1185 tablet_replica.cc:333] T df61317717d3401eb74cd93aa336c4dc P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:51.497928  1185 raft_consensus.cc:2440] T df61317717d3401eb74cd93aa336c4dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:51.498389  1185 raft_consensus.cc:2469] T df61317717d3401eb74cd93aa336c4dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:51.498513  1283 leader_election.cc:290] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:51.500200  1180 tablet_bootstrap.cc:654] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:51.500424  1185 ts_tablet_manager.cc:1962] T df61317717d3401eb74cd93aa336c4dc P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:51.502569  1295 ts_tablet_manager.cc:1962] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:51.488430  1324 consensus_queue.cc:261] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.510838  1324 ts_tablet_manager.cc:1480] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.038s	user 0.007s	sys 0.001s
I20261120 03:00:51.511756  1324 tablet_bootstrap.cc:492] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
W20261120 03:00:51.524814   487 catalog_manager.cc:4863] Async tablet task AlterTable RPC for tablet 88b9e9692a874ae993dde4b9e2586a70 (table default.Table_237971f3240b4730a6b7716296374a46 [id=b30488f6d14b4c9ab1566f2376be602a], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 88b9e9692a874ae993dde4b9e2586a70 has no consensus state
I20261120 03:00:51.528502  1324 tablet_bootstrap.cc:654] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:51.532061  1280 raft_consensus.cc:534] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:51.532517  1280 raft_consensus.cc:556] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.534224  1180 tablet_bootstrap.cc:492] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:51.534363  1280 leader_election.cc:290] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:51.534610  1180 ts_tablet_manager.cc:1449] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.045s	user 0.011s	sys 0.000s
I20261120 03:00:51.537640   497 master_service.cc:965] ReplaceTablet: received request to replace tablet 5f5c8711a6eb43c3aff1c27ed5115d6a from {username='slave'} at 127.0.0.1:45640
I20261120 03:00:51.537773  1180 raft_consensus.cc:366] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.538369  1180 raft_consensus.cc:392] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:51.538594  1180 raft_consensus.cc:787] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:51.538851   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5fee8dff291c43e9be0fb10b308718ce" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
W20261120 03:00:51.540170   857 leader_election.cc:343] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:51.540704  1353 consensus_queue.cc:1049] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:51.552423  1180 consensus_queue.cc:261] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.555696  1180 ts_tablet_manager.cc:1480] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.021s	user 0.005s	sys 0.000s
I20261120 03:00:51.556555  1180 tablet_bootstrap.cc:492] T a1f6197c60dd4dfb8aeb5c38373aab21 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:51.565485  1324 tablet_bootstrap.cc:492] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:51.565943  1324 ts_tablet_manager.cc:1449] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.054s	user 0.008s	sys 0.003s
I20261120 03:00:51.577798  1180 tablet_bootstrap.cc:654] T a1f6197c60dd4dfb8aeb5c38373aab21 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:51.578164  1185 ts_tablet_manager.cc:1975] T df61317717d3401eb74cd93aa336c4dc P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:51.578532  1185 log.cc:1199] T df61317717d3401eb74cd93aa336c4dc P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/df61317717d3401eb74cd93aa336c4dc
I20261120 03:00:51.579222  1185 ts_tablet_manager.cc:1996] T df61317717d3401eb74cd93aa336c4dc P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:51.580080  1324 raft_consensus.cc:366] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.580734  1324 raft_consensus.cc:392] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:51.580987  1324 raft_consensus.cc:787] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:51.581725  1324 consensus_queue.cc:261] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.583822  1324 ts_tablet_manager.cc:1480] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.018s	user 0.005s	sys 0.000s
I20261120 03:00:51.596799  1324 tablet_bootstrap.cc:492] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:51.598003   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet df61317717d3401eb74cd93aa336c4dc (table default.Table_237971f3240b4730a6b7716296374a46 [id=b30488f6d14b4c9ab1566f2376be602a]) successfully deleted
I20261120 03:00:51.599131  1295 ts_tablet_manager.cc:1975] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:51.600888   498 master_service.cc:965] ReplaceTablet: received request to replace tablet 3f65ba863447413cb122e728df3632fd from {username='slave'} at 127.0.0.1:45640
I20261120 03:00:51.603713  1324 tablet_bootstrap.cc:654] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:51.606503  1295 log.cc:1199] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/c3b56d0ccab140c28a37808d9d5902c6
I20261120 03:00:51.607429  1295 ts_tablet_manager.cc:1996] T c3b56d0ccab140c28a37808d9d5902c6 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:51.610299   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet c3b56d0ccab140c28a37808d9d5902c6 (table default.Table_84d4c48133054235908348e9d8cf1f2a [id=5312622ebb1347f78430eab828632d38]) successfully deleted
I20261120 03:00:51.615443  1295 tablet_replica.cc:333] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:51.617914  1295 raft_consensus.cc:2440] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:51.626456  1295 raft_consensus.cc:2469] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:51.629453  1295 ts_tablet_manager.cc:1962] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:51.632323  1324 tablet_bootstrap.cc:492] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:51.632750  1324 ts_tablet_manager.cc:1449] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.036s	user 0.014s	sys 0.000s
I20261120 03:00:51.635334  1324 raft_consensus.cc:366] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.636003  1324 raft_consensus.cc:392] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:51.636271  1324 raft_consensus.cc:787] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:51.637099  1324 consensus_queue.cc:261] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.639590  1324 ts_tablet_manager.cc:1480] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.007s	user 0.003s	sys 0.003s
I20261120 03:00:51.638062  1180 tablet_bootstrap.cc:492] T a1f6197c60dd4dfb8aeb5c38373aab21 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:51.640168  1180 ts_tablet_manager.cc:1449] T a1f6197c60dd4dfb8aeb5c38373aab21 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.084s	user 0.006s	sys 0.007s
I20261120 03:00:51.640506  1324 tablet_bootstrap.cc:492] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:51.643337  1180 raft_consensus.cc:366] T a1f6197c60dd4dfb8aeb5c38373aab21 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.643990  1180 raft_consensus.cc:392] T a1f6197c60dd4dfb8aeb5c38373aab21 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:51.644349  1180 raft_consensus.cc:787] T a1f6197c60dd4dfb8aeb5c38373aab21 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:51.645006  1180 consensus_queue.cc:261] T a1f6197c60dd4dfb8aeb5c38373aab21 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.647585   500 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_f73e1375172b4e9999e9906c1491c0d5"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:51.649348   500 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_f73e1375172b4e9999e9906c1491c0d5 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:51.658182  1289 raft_consensus.cc:534] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:51.658690  1289 raft_consensus.cc:556] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.660702  1289 leader_election.cc:290] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:51.661465  1324 tablet_bootstrap.cc:654] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:51.646809  1280 raft_consensus.cc:534] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:51.666323  1280 raft_consensus.cc:556] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:51.668202  1280 leader_election.cc:290] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:51.662344   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5fee8dff291c43e9be0fb10b308718ce" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:51.668903   923 raft_consensus.cc:2062] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:51.674908   710 leader_election.cc:304] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:51.676119  1289 raft_consensus.cc:3024] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:51.676446  1289 raft_consensus.cc:534] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:51.676584  1180 ts_tablet_manager.cc:1480] T a1f6197c60dd4dfb8aeb5c38373aab21 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.036s	user 0.004s	sys 0.001s
I20261120 03:00:51.676753  1289 raft_consensus.cc:3296] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:51.677284   499 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 3f65ba863447413cb122e728df3632fd
I20261120 03:00:51.677421  1180 tablet_bootstrap.cc:492] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:51.677480  1289 raft_consensus.cc:556] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:51.680207   857 leader_election.cc:343] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: df61317717d3401eb74cd93aa336c4dc
I20261120 03:00:51.684839  1289 leader_election.cc:290] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:51.685949   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5fee8dff291c43e9be0fb10b308718ce" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:51.686543   923 raft_consensus.cc:3296] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:51.687388   923 raft_consensus.cc:2062] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:51.691381   756 tablet_service.cc:1542] Processing DeleteTablet for tablet 3f65ba863447413cb122e728df3632fd with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:51 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:51.692034   754 tablet_service.cc:1542] Processing DeleteTablet for tablet a72d5d7896de4020a1ac4d2f8c551e15 with delete_type TABLET_DATA_DELETED (replaced by tablet ce612f9a121b4b18908724d2726e156f) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:51.692646   753 tablet_service.cc:1542] Processing DeleteTablet for tablet 136056e5242a42ef8fa4b40c989b5423 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:51.693363   755 tablet_service.cc:1542] Processing DeleteTablet for tablet d753dd170cab4f18980e85707c4093e9 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:51.694329  1185 tablet_replica.cc:333] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:51.695091  1185 raft_consensus.cc:2440] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:51.695541  1185 raft_consensus.cc:2469] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:51.698176   900 tablet_service.cc:1542] Processing DeleteTablet for tablet 3f65ba863447413cb122e728df3632fd with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:51 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:51.709956  1324 tablet_bootstrap.cc:492] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:51.710101  1185 ts_tablet_manager.cc:1962] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:51.710433  1324 ts_tablet_manager.cc:1449] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.070s	user 0.006s	sys 0.005s
I20261120 03:00:51.712898  1324 raft_consensus.cc:366] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.713533  1324 raft_consensus.cc:392] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:51.713791  1324 raft_consensus.cc:787] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:51.715518  1180 tablet_bootstrap.cc:654] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:51.715581  1012 consensus_peers.cc:597] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 -> Peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Couldn't send request to peer 3d30c2428730481db9b0ffd279dd3440. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c3b56d0ccab140c28a37808d9d5902c6. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:51.716356  1324 consensus_queue.cc:261] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.718315  1324 ts_tablet_manager.cc:1480] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.008s	user 0.006s	sys 0.000s
I20261120 03:00:51.719542  1324 tablet_bootstrap.cc:492] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:51.715417   710 leader_election.cc:304] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:51.725085  1355 raft_consensus.cc:3024] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:51.725548  1355 raft_consensus.cc:744] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:51.726361  1355 consensus_queue.cc:238] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.731119   502 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_5b0253debe2842c9a353d5152c3af3f6"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:51.732913  1324 tablet_bootstrap.cc:654] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:51.732805   502 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_5b0253debe2842c9a353d5152c3af3f6 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:00:51.749111   860 consensus_peers.cc:183] SignalRequest failed: Illegal state: peer closed
I20261120 03:00:51.751727  1280 raft_consensus.cc:534] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:51.752188  1280 raft_consensus.cc:556] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.754282  1280 leader_election.cc:290] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:51.755302   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5a3fe1499d0b4fab9f9949c8ded0a8dc" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:51.755291   493 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_237971f3240b4730a6b7716296374a46" } new_table_name: "default.Table_e63a7bef412545baac9d0cae0cf299ea" modify_external_catalogs: true
W20261120 03:00:51.756603   857 leader_election.cc:343] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:51.758809  1295 ts_tablet_manager.cc:1975] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:51.764420  1324 tablet_bootstrap.cc:492] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:51.764856  1324 ts_tablet_manager.cc:1449] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.045s	user 0.011s	sys 0.005s
W20261120 03:00:51.769418   495 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.66:36141 (request call id 20) took 819 ms (client timeout 1000 ms). Trace:
I20261120 03:00:51.768615  1324 raft_consensus.cc:366] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.770489  1324 raft_consensus.cc:392] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:51.770771  1324 raft_consensus.cc:787] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:51.771328  1324 consensus_queue.cc:261] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.773139  1324 ts_tablet_manager.cc:1480] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.008s	user 0.005s	sys 0.000s
W20261120 03:00:51.769886   495 rpcz_store.cc:269] 1120 03:00:50.950477 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:50.957362 (+  6885us) service_pool.cc:224] Handling call
1120 03:00:51.769353 (+811991us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:51.275529 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:51.275970 (+   441us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:51.275994 (+    24us) write_op.cc:440] Acquired schema lock
1120 03:00:51.276007 (+    13us) tablet.cc:710] Decoding operations
1120 03:00:51.276281 (+   274us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:51.276324 (+    43us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:51.276345 (+    21us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:00:51.276492 (+   147us) tablet.cc:758] Row locks acquired
1120 03:00:51.276504 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:00:51.276641 (+   137us) write_op.cc:270] Start()
1120 03:00:51.276725 (+    84us) write_op.cc:276] Timestamp: P: 1795143651276569 usec, L: 0
1120 03:00:51.276740 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:00:51.277211 (+   471us) log.cc:844] Serialized 355 byte log entry
1120 03:00:51.291351 (+ 14140us) op_driver.cc:466] REPLICATION: finished
1120 03:00:51.291587 (+   236us) write_op.cc:301] APPLY: starting
1120 03:00:51.291637 (+    50us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:51.291731 (+    94us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:51.291828 (+    97us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:51.291995 (+   167us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:51.292114 (+   119us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:51.292156 (+    42us) write_op.cc:312] APPLY: finished
1120 03:00:51.292352 (+   196us) log.cc:844] Serialized 22 byte log entry
1120 03:00:51.292477 (+   125us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:51.292539 (+    62us) write_op.cc:459] Released schema lock
1120 03:00:51.292579 (+    40us) write_op.cc:346] FINISH: Updating metrics
1120 03:00:51.295969 (+  3390us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:51.295992 (+    23us) write_op.cc:459] Released schema lock
Metrics: {"reported_tablets":9,"child_traces":[["op",{"apply.queue_time_us":81,"apply.run_cpu_time_us":1200,"apply.run_wall_time_us":1199,"num_ops":1,"prepare.queue_time_us":265739,"prepare.run_cpu_time_us":2029,"prepare.run_wall_time_us":20431,"raft.queue_time_us":783,"raft.run_cpu_time_us":774,"raft.run_wall_time_us":1897,"replication_time_us":14521}]]}
I20261120 03:00:51.782475   495 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_9498dfca4c8544c0a19cd826d27d441e"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:51.786394   712 consensus_peers.cc:183] SignalRequest failed: Illegal state: peer closed
I20261120 03:00:51.769145   901 tablet_service.cc:1542] Processing DeleteTablet for tablet c3b56d0ccab140c28a37808d9d5902c6 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:51.769415   900 tablet_service.cc:1542] Processing DeleteTablet for tablet a642c32c1b6043978516d97efa2347a3 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:51.769625   899 tablet_service.cc:1542] Processing DeleteTablet for tablet 06c3bac919e342b09dd6af13aebc387b with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:51.770633  1180 tablet_bootstrap.cc:492] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:51.772186   903 tablet_service.cc:1542] Processing DeleteTablet for tablet 136056e5242a42ef8fa4b40c989b5423 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:51.772471   902 tablet_service.cc:1542] Processing DeleteTablet for tablet a72d5d7896de4020a1ac4d2f8c551e15 with delete_type TABLET_DATA_DELETED (replaced by tablet ce612f9a121b4b18908724d2726e156f) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:51.787842  1180 ts_tablet_manager.cc:1449] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.111s	user 0.007s	sys 0.004s
I20261120 03:00:51.789003  1295 log.cc:1199] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/d753dd170cab4f18980e85707c4093e9
I20261120 03:00:51.790772  1180 raft_consensus.cc:366] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.791419  1180 raft_consensus.cc:392] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:51.791662  1180 raft_consensus.cc:787] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:51.792258  1355 consensus_queue.cc:1049] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:51.792277  1180 consensus_queue.cc:261] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:51.794617  1180 ts_tablet_manager.cc:1480] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.006s	user 0.001s	sys 0.005s
I20261120 03:00:51.817025   497 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 5f5c8711a6eb43c3aff1c27ed5115d6a
I20261120 03:00:51.821940   497 catalog_manager.cc:6952] ReplaceTablet: tablet 5f5c8711a6eb43c3aff1c27ed5115d6a deleted and replaced by tablet a1854e4d9016414ea5c086ad56224b1f
I20261120 03:00:51.823719   755 tablet_service.cc:1542] Processing DeleteTablet for tablet 5f5c8711a6eb43c3aff1c27ed5115d6a with delete_type TABLET_DATA_DELETED (replaced by tablet a1854e4d9016414ea5c086ad56224b1f) from {username='slave'} at 127.0.0.1:52644
W20261120 03:00:51.836266   495 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_9498dfca4c8544c0a19cd826d27d441e in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:51.827085   902 tablet_service.cc:1542] Processing DeleteTablet for tablet 5f5c8711a6eb43c3aff1c27ed5115d6a with delete_type TABLET_DATA_DELETED (replaced by tablet a1854e4d9016414ea5c086ad56224b1f) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:51.798121  1295 ts_tablet_manager.cc:1996] T d753dd170cab4f18980e85707c4093e9 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:51.878211   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet d753dd170cab4f18980e85707c4093e9 (table default.Table_269fbc3b47a44895be4b7bebddeee096 [id=62fab6f5923b4aa3be379f3d3d8d3046]) successfully deleted
I20261120 03:00:51.897759  1185 ts_tablet_manager.cc:1975] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
W20261120 03:00:51.902905  1300 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
W20261120 03:00:51.903728  1300 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
I20261120 03:00:51.903874   502 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_0a4ee63ac2c442c5bae84e199b8a9664" } modify_external_catalogs: true
W20261120 03:00:51.904505  1300 flags.cc:432] Enabled unsafe flag: --never_fsync=true
I20261120 03:00:51.904568   502 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_0a4ee63ac2c442c5bae84e199b8a9664" } modify_external_catalogs: true
I20261120 03:00:51.910991  1185 log.cc:1199] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/3f65ba863447413cb122e728df3632fd
I20261120 03:00:51.912200  1185 ts_tablet_manager.cc:1996] T 3f65ba863447413cb122e728df3632fd P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:51.916381   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 3f65ba863447413cb122e728df3632fd (table default.Table_4755bb4a11ae4e33bff2a6cc7384921a [id=676ae8fc1a8c488d820a11b930b0ed5f]) successfully deleted
I20261120 03:00:51.916606  1375 tablet.cc:1872] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20261120 03:00:51.922884  1376 tablet.cc:1872] T 1237b0d2fa414ef38271835b5d69d6b9 P 3d30c2428730481db9b0ffd279dd3440: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
W20261120 03:00:51.923620   486 catalog_manager.cc:5048] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet a72d5d7896de4020a1ac4d2f8c551e15 because the tablet was not found. No further retry: Not found: Tablet not found: a72d5d7896de4020a1ac4d2f8c551e15
W20261120 03:00:51.925863   486 catalog_manager.cc:5048] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 136056e5242a42ef8fa4b40c989b5423 because the tablet was not found. No further retry: Not found: Tablet not found: 136056e5242a42ef8fa4b40c989b5423
W20261120 03:00:51.928155   486 catalog_manager.cc:5048] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet d753dd170cab4f18980e85707c4093e9 because the tablet was not found. No further retry: Not found: Tablet not found: d753dd170cab4f18980e85707c4093e9
I20261120 03:00:51.929484  1185 tablet_replica.cc:333] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:51.930971  1185 raft_consensus.cc:2440] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:51.935740  1185 raft_consensus.cc:2469] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:51.937484  1295 tablet_replica.cc:333] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:51.942090  1295 raft_consensus.cc:2440] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:51.942551  1295 raft_consensus.cc:2469] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:51.946504   502 catalog_manager.cc:6146] Not sending DeleteTablet requests; no consensus state for tablet a1854e4d9016414ea5c086ad56224b1f
I20261120 03:00:51.948941  1185 ts_tablet_manager.cc:1962] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:51.964934  1295 ts_tablet_manager.cc:1962] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
W20261120 03:00:51.977262   711 consensus_peers.cc:183] SignalRequest failed: Illegal state: peer closed
W20261120 03:00:51.992730   486 consensus_peers.cc:597] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 -> Peer 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133): Couldn't send request to peer 2599fd9eef7d4ce990c40e2356a37c3f. Status: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20261120 03:00:52.015365   902 tablet_service.cc:1495] Processing CreateTablet for tablet 200b234da897419a82bf5844620adf24 (DEFAULT_TABLE table=default.Table_5303db2882934be1bfc2251ce51f2c8d [id=ae52d3d892e24bc38a8ff7d6db755c2d]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.016498   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 200b234da897419a82bf5844620adf24. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.023321   498 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_3d732539a5434ae193267b36578eb1c3"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:52.025060   498 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_3d732539a5434ae193267b36578eb1c3 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:52.038663   494 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 1237b0d2fa414ef38271835b5d69d6b9 (table default.Table_523caa128be441e1a52767e3d3318ce0 [id=93a0027d82064a208e2535349e9b042d]). Expected version 1 got 0
I20261120 03:00:52.039381   502 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_5b0253debe2842c9a353d5152c3af3f6" } new_table_name: "default.Table_04ddab7f706b43e7ba12fa247b3ca933" modify_external_catalogs: true
I20261120 03:00:52.039520   494 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 5fee8dff291c43e9be0fb10b308718ce (table default.Table_a1f866996679431e9e1c4c5f6f8cdd12 [id=a275e215155b4824a1494f0954577fe8]). Expected version 1 got 0
I20261120 03:00:52.040517   494 catalog_manager.cc:5856] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65). New cstate: current_term: 1 leader_uuid: "5beeefe8bef54caeac463d05d8c30fa6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } }
I20261120 03:00:52.054490   903 tablet_service.cc:1495] Processing CreateTablet for tablet d27099bc667b4a1d880af7cec2715bb8 (DEFAULT_TABLE table=default.Table_d2f8156a63404ca5a92c6ea7190d15ad [id=767873bd6c694dcba7bc1df94dcf3925]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.055264   755 tablet_service.cc:1495] Processing CreateTablet for tablet 200b234da897419a82bf5844620adf24 (DEFAULT_TABLE table=default.Table_5303db2882934be1bfc2251ce51f2c8d [id=ae52d3d892e24bc38a8ff7d6db755c2d]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.055672   903 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d27099bc667b4a1d880af7cec2715bb8. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.056313   755 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 200b234da897419a82bf5844620adf24. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.046351   495 master_service.cc:965] ReplaceTablet: received request to replace tablet fa57d0c282964e37b9133ea57f458a51 from {username='slave'} at 127.0.0.1:45640
I20261120 03:00:52.067884   500 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_863c1a82fb1a4f2db6a422422dc64c04"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:52.069386   500 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_863c1a82fb1a4f2db6a422422dc64c04 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:52.071787   501 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_3adc64475e544d4e9c19ed6354bdc9be"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:52.073899   501 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_3adc64475e544d4e9c19ed6354bdc9be in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:00:52.078565   493 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 88b9e9692a874ae993dde4b9e2586a70 has no consensus state
I20261120 03:00:52.081678  1180 tablet_bootstrap.cc:492] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:52.084810   753 tablet_service.cc:1495] Processing CreateTablet for tablet d27099bc667b4a1d880af7cec2715bb8 (DEFAULT_TABLE table=default.Table_d2f8156a63404ca5a92c6ea7190d15ad [id=767873bd6c694dcba7bc1df94dcf3925]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.085875   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d27099bc667b4a1d880af7cec2715bb8. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.087378  1180 tablet_bootstrap.cc:654] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:52.094635   899 tablet_service.cc:1495] Processing CreateTablet for tablet 9882c9cbfbd640a49e3e3c8904ffc1e1 (DEFAULT_TABLE table=default.Table_3ce972a9130f459f940b2910e9eb1427 [id=53fb1849755c4debad2dbfbf37271fbd]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.095901   899 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9882c9cbfbd640a49e3e3c8904ffc1e1. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.098801  1185 ts_tablet_manager.cc:1975] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20261120 03:00:52.099234  1185 log.cc:1199] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/5f5c8711a6eb43c3aff1c27ed5115d6a
I20261120 03:00:52.099968  1185 ts_tablet_manager.cc:1996] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:52.104266   754 tablet_service.cc:1495] Processing CreateTablet for tablet 33e6cd23a3474034953c9b1a37b79acd (DEFAULT_TABLE table=default.Table_8c3f3e4b96ba410fa0109e46c25d4144 [id=9bbb6f66b4a841ac8bd1253946003db0]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.106437   756 tablet_service.cc:1495] Processing CreateTablet for tablet 9882c9cbfbd640a49e3e3c8904ffc1e1 (DEFAULT_TABLE table=default.Table_3ce972a9130f459f940b2910e9eb1427 [id=53fb1849755c4debad2dbfbf37271fbd]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.107097   754 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 33e6cd23a3474034953c9b1a37b79acd. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.107896   756 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9882c9cbfbd640a49e3e3c8904ffc1e1. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.111796  1180 tablet_bootstrap.cc:492] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:52.112215  1180 ts_tablet_manager.cc:1449] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.031s	user 0.011s	sys 0.000s
I20261120 03:00:52.114562  1180 raft_consensus.cc:366] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:52.115324  1180 raft_consensus.cc:392] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:52.115640  1180 raft_consensus.cc:787] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
==================
WARNING: ThreadSanitizer: data race (pid=977)
  Write of size 8 at 0x7b44000b4e48 by thread T127:
    #0 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::__destruct_at_end(kudu::MemTracker**) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:429:12 (libkudu_util.so+0x382770)
    #1 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libkudu_util.so+0x382684)
I20261120 03:00:52.121335  1180 consensus_queue.cc:261] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
    #2 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libkudu_util.so+0x38247b)
    #3 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libkudu_util.so+0x37a621)
    #4 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x37a1d3)
    #5 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x381495)
    #6 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libkudu_cli_actions.so+0x24d225)
    #7 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libkudu_cli_actions.so+0x24d1c9)
    #8 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #9 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x450864)
    #10 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x403c45)
    #11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x403e59)
    #12 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libkudu_cli_actions.so+0x2fbea7)
    #13 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libkudu_cli_actions.so+0x2fbe69)
    #14 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libkudu_cli_actions.so+0x2fbe29)
    #15 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libkudu_cli_actions.so+0x2fa1fa)
    #16 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3e1f09)
    #17 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3e1ee0)
    #18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3e1ea0)
    #19 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b4aa6)
    #20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b49c4)
    #21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b477b)
    #22 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x28c761)
    #23 kudu::tserver::TSTabletManager::PopulateIncrementalTabletReport(kudu::master::TabletReportPB*, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, bool) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1893:1 (libtserver.so+0x2c1b50)
    #24 kudu::tserver::Heartbeater::Thread::GenerateIncrementalTabletReport(kudu::master::TabletReportPB*, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:790:30 (libtserver.so+0x1cb58a)
    #25 kudu::tserver::Heartbeater::Thread::DoHeartbeat(kudu::master::MasterErrorPB*, kudu::rpc::ErrorStatusPB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:521:5 (libtserver.so+0x1cd4c4)
    #26 kudu::tserver::Heartbeater::Thread::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:645:21 (libtserver.so+0x1ce790)
    #27 kudu::tserver::Heartbeater::Thread::Start()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:728:48 (libtserver.so+0x1d03c1)
    #28 decltype(std::__1::forward<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(fp)()) std::__1::__invoke<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x1d0379)
    #29 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x1d0309)
    #30 std::__1::__function::__alloc_func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x1d02d1)
    #31 std::__1::__function::__func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x1cf5cd)
    #32 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #33 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #34 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Previous read of size 8 at 0x7b44000b4e48 by thread T144 (mutexes: write M299906313465758112):
    #0 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::end() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1536:30 (libkudu_util.so+0x37b53a)
    #1 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:284:22 (libkudu_util.so+0x378c16)
    #2 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x451230)
    #3 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:608:16 (libtablet.so+0x44a1c2)
    #4 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:579:5 (libtablet.so+0x449953)
    #5 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:65 (libtablet.so+0x44db61)
    #6 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x44db19)
    #7 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x44daa9)
    #8 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x44da71)
    #9 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x44cc9d)
    #10 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #11 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #12 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x4403a5)
    #13 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x442711)
    #14 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x4426c9)
    #15 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x442659)
    #16 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x442621)
I20261120 03:00:52.123251  1180 ts_tablet_manager.cc:1480] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.011s	user 0.002s	sys 0.003s
    #17 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x44191d)
    #18 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #19 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #20 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Location is heap block of size 264 at 0x7b44000b4dc0 allocated by thread T135:
    #0 operator new(unsigned long) /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:64 (kudu+0x320f47)
    #1 std::__1::__libcpp_allocate(unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:253:10 (libkudu_util.so+0x2ae876)
    #2 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::allocate(unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1789:34 (libkudu_util.so+0x380d75)
    #3 std::__1::enable_if<!(is_array<long&>::value), std::__1::shared_ptr<long&> >::type std::__1::make_shared<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler&&...) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4290:45 (libkudu_util.so+0x380b51)
    #4 std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/make_shared.h:61:12 (libkudu_util.so+0x37a485)
    #5 kudu::MemTracker::CreateTracker(long, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:54:34 (libkudu_util.so+0x3784ae)
    #6 kudu::tablet::OpTracker::StartMemoryTracking(std::__1::shared_ptr<kudu::MemTracker> const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:311:20 (libtablet.so+0x451ceb)
    #7 kudu::tablet::TabletReplica::Start(kudu::consensus::ConsensusBootstrapInfo const&, std::__1::shared_ptr<kudu::tablet::Tablet>, kudu::clock::Clock*, std::__1::shared_ptr<kudu::rpc::Messenger>, scoped_refptr<kudu::rpc::ResultTracker>, scoped_refptr<kudu::log::Log>, kudu::consensus::MultiRaftManager*, kudu::ThreadPool*, kudu::DnsResolver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:278:19 (libtablet.so+0x404ca6)
    #8 kudu::tserver::TSTabletManager::OpenTablet(scoped_refptr<kudu::tablet::TabletReplica> const&, scoped_refptr<kudu::tserver::TransitionInProgressDeleter> const&, std::__1::atomic<int>*, std::__1::atomic<int>*, kudu::Timer*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1482:18 (libtserver.so+0x2bd7ba)
    #9 kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:755:3 (libtserver.so+0x2d2901)
    #10 decltype(std::__1::forward<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12&>(fp)()) std::__1::__invoke<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12&>(kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x2d28a9)
    #11 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12&>(kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x2d2839)
    #12 std::__1::__function::__alloc_func<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12, std::__1::allocator<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x2d2801)
    #13 std::__1::__function::__func<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12, std::__1::allocator<kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*)::$_12>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x2d1add)
    #14 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #15 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #16 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x4403a5)
    #17 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x442711)
    #18 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x4426c9)
    #19 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x442659)
    #20 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x442621)
    #21 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x44191d)
    #22 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #23 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #24 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Mutex M299906313465758112 is already destroyed.

  Thread T127 'heartbeat-1124' (tid=1124, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::tserver::Heartbeater::Thread::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:727:10 (libtserver.so+0x1cae61)
    #4 kudu::tserver::Heartbeater::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:251:41 (libtserver.so+0x1cac3d)
    #5 kudu::tserver::TabletServer::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:167:3 (libtserver.so+0x259ccc)
    #6 kudu::tserver::RunTabletServer() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server_runner.cc:86:3 (libtserver.so+0x266e93)
    #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:128:10 (libkudu_cli_actions.so+0x3b6429)
    #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_cli_actions.so+0x28c051)
    #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (libkudu_cli_actions.so+0x28bfbc)
    #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_cli_actions.so+0x28bf2c)
    #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_cli_actions.so+0x28ab95)
    #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x14082f)
    #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132da1)
    #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:270:10 (libkudu_tools_util.so+0x130a52)
    #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:114:15 (kudu+0x324910)
    #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:186:16 (kudu+0x325247)
    #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:249:10 (kudu+0x325671)

  Thread T144 'apply [worker]-' (tid=1286, running) created by thread T82 at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x43f5cb)
    #4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x43d8cb)
    #5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x43fb32)
    #6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:23 (libtablet.so+0x448cf0)
    #7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:476:5 (libtablet.so+0x449289)
    #8 kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:860:15 (libtablet.so+0x41362a)
    #9 decltype(std::__1::forward<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x4135b8)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x413538)
    #11 std::__1::__function::__alloc_func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x4134f3)
    #12 std::__1::__function::__func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41270c)
    #13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x40fe06)
    #14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x40a5b8)
    #15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3598:3 (libconsensus.so+0x1fd9c0)
    #16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:200:12 (libconsensus.so+0x1d2e9d)
    #17 kudu::consensus::RaftConsensus::UpdateReplica(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::consensus::RaftConsensus::FlushContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1603:5 (libconsensus.so+0x1f0713)
    #18 kudu::consensus::RaftConsensus::Update(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1162:9 (libconsensus.so+0x1efd04)
    #19 kudu::tserver::ConsensusServiceImpl::UpdateConsensus(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1792:25 (libtserver.so+0x26e199)
    #20 kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/consensus/consensus.service.cc:321:13 (libconsensus_proto.so+0xd8d34)
    #21 decltype(std::__1::forward<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus_proto.so+0xd8cc2)
    #22 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus_proto.so+0xd8bf1)
    #23 std::__1::__function::__alloc_func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus_proto.so+0xd8b6c)
    #24 std::__1::__function::__func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus_proto.so+0xd7e22)
    #25 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x21b0bc)
    #26 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x21a4f6)
    #27 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x219e9f)
    #28 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15 (libkrpc.so+0x21d16f)
    #29 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5 (libkrpc.so+0x21e4c1)
    #30 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x21e479)
    #31 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x21e409)
    #32 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x21e3d1)
    #33 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x21d6cd)
    #34 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #35 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #36 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Thread T135 'tablet-open [wo' (tid=1181, running) created by thread T62 at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x43f5cb)
    #4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x43d8cb)
    #5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x43fb32)
    #6 kudu::tserver::TSTabletManager::CreateNewTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Partition const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::Schema const&, kudu::PartitionSchema const&, kudu::consensus::RaftConfigPB, std::__1::optional<kudu::TableExtraConfigPB>, std::__1::optional<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::optional<kudu::TableTypePB>, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:755:3 (libtserver.so+0x2ba982)
    #7 kudu::tserver::TabletServiceAdminImpl::CreateTablet(kudu::tserver::CreateTabletRequestPB const*, kudu::tserver::CreateTabletResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1502:34 (libtserver.so+0x26a180)
    #8 kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/tserver/tserver_admin.service.cc:189:13 (libtserver_admin_proto.so+0x68bf4)
    #9 decltype(std::__1::forward<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver_admin_proto.so+0x68b82)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver_admin_proto.so+0x68ab1)
    #11 std::__1::__function::__alloc_func<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver_admin_proto.so+0x68a2c)
    #12 std::__1::__function::__func<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::tserver::TabletServerAdminServiceIf::TabletServerAdminServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver_admin_proto.so+0x67ce2)
    #13 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x21b0bc)
    #14 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x21a4f6)
    #15 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x219e9f)
    #16 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15 (libkrpc.so+0x21d16f)
    #17 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5 (libkrpc.so+0x21e4c1)
    #18 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x21e479)
    #19 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x21e409)
    #20 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x21e3d1)
    #21 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x21d6cd)
    #22 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #23 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #24 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:429:12 in std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::__destruct_at_end(kudu::MemTracker**)
I20261120 03:00:52.134861   493 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_eccdaea2421f42fd95a4253870c13623"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
==================
W20261120 03:00:52.136616   493 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_eccdaea2421f42fd95a4253870c13623 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:52.138149  1180 tablet_bootstrap.cc:492] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:52.142805  1295 ts_tablet_manager.cc:1975] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:52.143183  1295 log.cc:1199] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/df61317717d3401eb74cd93aa336c4dc
I20261120 03:00:52.143834  1295 ts_tablet_manager.cc:1996] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:52.143982  1180 tablet_bootstrap.cc:654] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:52.145649   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet df61317717d3401eb74cd93aa336c4dc (table default.Table_e63a7bef412545baac9d0cae0cf299ea [id=b30488f6d14b4c9ab1566f2376be602a]) successfully deleted
W20261120 03:00:52.147809  1300 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
I20261120 03:00:52.149554   497 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_58a2bd95297c48189f4c9338a266cabd" } modify_external_catalogs: true
I20261120 03:00:52.150121   497 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_58a2bd95297c48189f4c9338a266cabd" } modify_external_catalogs: true
W20261120 03:00:52.157222   502 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet d736d65c3e0b419084b4934dad83f16a has no consensus state
W20261120 03:00:52.159053  1300 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
W20261120 03:00:52.159482  1300 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
W20261120 03:00:52.159899  1300 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
==================
WARNING: ThreadSanitizer: data race (pid=977)
  Write of size 8 at 0x7b08000d5460 by thread T127:
    #0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (kudu+0x321819)
    #1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (kudu+0x3287c9)
    #2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (kudu+0x328769)
    #3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x2aebd2)
    #4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x2aeb69)
    #5 std::__1::allocator<kudu::MemTracker*>::deallocate(kudu::MemTracker**, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x38284a)
    #6 std::__1::allocator_traits<std::__1::allocator<kudu::MemTracker*> >::deallocate(std::__1::allocator<kudu::MemTracker*>&, kudu::MemTracker**, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1533:14 (libkudu_util.so+0x3826c9)
    #7 std::__1::__vector_base<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:466:9 (libkudu_util.so+0x3824a7)
    #8 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libkudu_util.so+0x37a621)
    #9 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x37a1d3)
    #10 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x381495)
    #11 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libkudu_cli_actions.so+0x24d225)
    #12 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libkudu_cli_actions.so+0x24d1c9)
    #13 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #14 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x450864)
    #15 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x403c45)
    #16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x403e59)
    #17 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libkudu_cli_actions.so+0x2fbea7)
    #18 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libkudu_cli_actions.so+0x2fbe69)
    #19 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libkudu_cli_actions.so+0x2fbe29)
    #20 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libkudu_cli_actions.so+0x2fa1fa)
    #21 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3e1f09)
    #22 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3e1ee0)
    #23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3e1ea0)
    #24 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b4aa6)
    #25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b49c4)
    #26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b477b)
    #27 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x28c761)
    #28 kudu::tserver::TSTabletManager::PopulateIncrementalTabletReport(kudu::master::TabletReportPB*, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, bool) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1893:1 (libtserver.so+0x2c1b50)
    #29 kudu::tserver::Heartbeater::Thread::GenerateIncrementalTabletReport(kudu::master::TabletReportPB*, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:790:30 (libtserver.so+0x1cb58a)
    #30 kudu::tserver::Heartbeater::Thread::DoHeartbeat(kudu::master::MasterErrorPB*, kudu::rpc::ErrorStatusPB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:521:5 (libtserver.so+0x1cd4c4)
    #31 kudu::tserver::Heartbeater::Thread::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:645:21 (libtserver.so+0x1ce790)
    #32 kudu::tserver::Heartbeater::Thread::Start()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:728:48 (libtserver.so+0x1d03c1)
    #33 decltype(std::__1::forward<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(fp)()) std::__1::__invoke<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x1d0379)
    #34 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x1d0309)
    #35 std::__1::__function::__alloc_func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x1d02d1)
    #36 std::__1::__function::__func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x1cf5cd)
    #37 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #38 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #39 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Previous read of size 8 at 0x7b08000d5460 by thread T144 (mutexes: write M299906313465758112):
    #0 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:5 (libkudu_util.so+0x378c52)
    #1 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x451230)
    #2 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:608:16 (libtablet.so+0x44a1c2)
    #3 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:579:5 (libtablet.so+0x449953)
    #4 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:65 (libtablet.so+0x44db61)
    #5 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x44db19)
    #6 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x44daa9)
    #7 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x44da71)
    #8 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x44cc9d)
I20261120 03:00:52.177731  1280 raft_consensus.cc:534] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
    #9 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #10 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #11 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x4403a5)
    #12 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x442711)
    #13 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x4426c9)
    #14 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x442659)
    #15 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x442621)
    #16 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x44191d)
    #17 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #18 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #19 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Mutex M299906313465758112 is already destroyed.

  Thread T127 'heartbeat-1124' (tid=1124, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::tserver::Heartbeater::Thread::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:727:10 (libtserver.so+0x1cae61)
    #4 kudu::tserver::Heartbeater::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:251:41 (libtserver.so+0x1cac3d)
    #5 kudu::tserver::TabletServer::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:167:3 (libtserver.so+0x259ccc)
    #6 kudu::tserver::RunTabletServer() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server_runner.cc:86:3 (libtserver.so+0x266e93)
    #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:128:10 (libkudu_cli_actions.so+0x3b6429)
    #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_cli_actions.so+0x28c051)
    #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (libkudu_cli_actions.so+0x28bfbc)
    #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_cli_actions.so+0x28bf2c)
    #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_cli_actions.so+0x28ab95)
    #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x14082f)
    #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132da1)
    #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:270:10 (libkudu_tools_util.so+0x130a52)
    #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:114:15 (kudu+0x324910)
    #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:186:16 (kudu+0x325247)
    #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:249:10 (kudu+0x325671)

  Thread T144I20261120 03:00:52.178210  1280 raft_consensus.cc:556] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
 'apply [worker]-' (tid=1286, running) created by thread T82 at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x43f5cb)
    #4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x43d8cb)
    #5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x43fb32)
    #6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:23 (libtablet.so+0x448cf0)
    #7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:476:5 (libtablet.so+0x449289)
    #8 kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:860:15 (libtablet.so+0x41362a)
    #9 decltype(std::__1::forward<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x4135b8)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x413538)
    #11 std::__1::__function::__alloc_func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x4134f3)
    #12 std::__1::__function::__func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41270c)
    #13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x40fe06)
    #14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x40a5b8)
    #15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3598:3 (libconsensus.so+0x1fd9c0)
    #16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:200:12 (libconsensus.so+0x1d2e9d)
    #17 kudu::consensus::RaftConsensus::UpdateReplica(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::consensus::RaftConsensus::FlushContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1603:5 (libconsensus.so+0x1f0713)
    #18 kudu::consensus::RaftConsensus::Update(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1162:9 (libconsensus.so+0x1efd04)
    #19 kudu::tserver::ConsensusServiceImpl::UpdateConsensus(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1792:25 (libtserver.so+0x26e199)
    #20 kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/consensus/consensus.service.cc:321:13 (libconsensus_proto.so+0xd8d34)
    #21 decltype(std::__1::forward<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus_proto.so+0xd8cc2)
    #22 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus_proto.so+0xd8bf1)
    #23 std::__1::__function::__alloc_func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus_proto.so+0xd8b6c)
    #24 std::__1::__function::__func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus_proto.so+0xd7e22)
    #25 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x21b0bc)
    #26 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x21a4f6)
    #27 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x219e9f)
    #28 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15 (libkrpc.so+0x21d16f)
    #29 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5 (libkrpc.so+0x21e4c1)
    #30 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x21e479)
    #31 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x21e409)
    #32 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x21e3d1)
    #33 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x21d6cd)
    #34 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #35 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #36 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
==================
I20261120 03:00:52.180065  1280 leader_election.cc:290] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:52.182034   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9a31431495274c648625b75900b4ac47" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:52.182758   777 raft_consensus.cc:2062] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:52.184226   857 leader_election.cc:304] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:52.185189  1280 raft_consensus.cc:3024] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:52.185638  1280 raft_consensus.cc:534] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:52.185986  1280 raft_consensus.cc:3296] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:52.191520  1180 tablet_bootstrap.cc:492] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:52.191987  1180 ts_tablet_manager.cc:1449] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.054s	user 0.007s	sys 0.003s
I20261120 03:00:52.193683  1289 raft_consensus.cc:534] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:52.194145  1289 raft_consensus.cc:556] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:52.196174  1289 leader_election.cc:290] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:52.201087   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a1f6197c60dd4dfb8aeb5c38373aab21" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:52.201861   923 raft_consensus.cc:2062] T a1f6197c60dd4dfb8aeb5c38373aab21 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:52.202742  1295 tablet_replica.cc:333] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
W20261120 03:00:52.203387  1181 thread.cc:642] log-alloc [worker] (thread pool) Time spent creating pthread: real 1.487s	user 0.003s	sys 0.002s
W20261120 03:00:52.203691  1181 thread.cc:609] log-alloc [worker] (thread pool) Time spent starting thread: real 1.487s	user 0.003s	sys 0.002s
I20261120 03:00:52.207072  1295 raft_consensus.cc:2440] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:52.207939  1295 raft_consensus.cc:2469] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:52.210798  1289 raft_consensus.cc:534] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:52.211120   710 leader_election.cc:304] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:52.211208  1180 raft_consensus.cc:366] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:52.211256  1289 raft_consensus.cc:556] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:52.211414   902 tablet_service.cc:1495] Processing CreateTablet for tablet 020be7f7cda0467aab56f657cbe8d0ad (DEFAULT_TABLE table=default.Table_d1737969bdd644a6ab39da121eeedd60 [id=ed325a58ff124ba9a6b1d05bc901793b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.211822  1180 raft_consensus.cc:392] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:52.211941  1355 raft_consensus.cc:3024] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:52.212107  1180 raft_consensus.cc:787] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:52.212270  1355 raft_consensus.cc:534] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:52.212556   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 020be7f7cda0467aab56f657cbe8d0ad. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.212560  1355 raft_consensus.cc:3296] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:52.213271  1355 raft_consensus.cc:556] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:52.215325   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "4abb67f3819a45b9b05dde04842e2c9d" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:52.216019   923 raft_consensus.cc:2062] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:52.216965  1289 leader_election.cc:290] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:52.217689   710 leader_election.cc:304] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:52.218669  1289 raft_consensus.cc:3024] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:52.219084  1289 raft_consensus.cc:534] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:52.219389  1289 raft_consensus.cc:3296] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
W20261120 03:00:52.230090   859 consensus_peers.cc:183] SignalRequest failed: Illegal state: peer closed
I20261120 03:00:52.232422  1324 tablet_bootstrap.cc:492] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:52.236994  1384 raft_consensus.cc:534] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:52.238742  1388 raft_consensus.cc:534] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:52.239143   903 tablet_service.cc:1495] Processing CreateTablet for tablet 33e6cd23a3474034953c9b1a37b79acd (DEFAULT_TABLE table=default.Table_8c3f3e4b96ba410fa0109e46c25d4144 [id=9bbb6f66b4a841ac8bd1253946003db0]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.239694  1181 tablet_bootstrap.cc:492] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:52.239333  1388 raft_consensus.cc:556] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:52.240190  1181 ts_tablet_manager.cc:1449] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 1.599s	user 0.007s	sys 0.005s
I20261120 03:00:52.240528   903 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 33e6cd23a3474034953c9b1a37b79acd. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.241444  1388 leader_election.cc:290] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:52.242861  1181 raft_consensus.cc:366] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:52.243643  1181 raft_consensus.cc:392] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:52.243947  1181 raft_consensus.cc:787] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:52.244910  1056 tablet_service.cc:1495] Processing CreateTablet for tablet 4abb67f3819a45b9b05dde04842e2c9d (DEFAULT_TABLE table=default.Table_69539e86f82e4df2a4b6888aa55293fc [id=0335c1c0df984b0ab347ae9e6d566c5d]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.245030  1181 consensus_queue.cc:261] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:52.246740  1056 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4abb67f3819a45b9b05dde04842e2c9d. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.248389  1295 ts_tablet_manager.cc:1962] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:52.251120  1058 tablet_service.cc:1495] Processing CreateTablet for tablet a1f6197c60dd4dfb8aeb5c38373aab21 (DEFAULT_TABLE table=default.Table_0b2a37ca199f431fb72c9bb9db84eef3 [id=6a353839c63741c3a49ec2e5724283f5]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.252321  1058 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a1f6197c60dd4dfb8aeb5c38373aab21. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.254992  1059 tablet_service.cc:1542] Processing DeleteTablet for tablet a72d5d7896de4020a1ac4d2f8c551e15 with delete_type TABLET_DATA_DELETED (replaced by tablet ce612f9a121b4b18908724d2726e156f) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:52.255028   899 tablet_service.cc:1495] Processing CreateTablet for tablet fbbe19dadb08455285078e239d81bcca (DEFAULT_TABLE table=default.Table_85ad53c3650a452b9c9ab5d515add49c [id=a8f19086139642fab5626a91bdaa6713]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.256209   899 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fbbe19dadb08455285078e239d81bcca. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.257309  1057 tablet_service.cc:1495] Processing CreateTablet for tablet 5a3fe1499d0b4fab9f9949c8ded0a8dc (DEFAULT_TABLE table=default.Table_14745a85367e453183d6a0a197be3fbb [id=c27784e6e0c84353a8aec497dd69b9ff]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.258538  1057 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5a3fe1499d0b4fab9f9949c8ded0a8dc. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.262215  1055 tablet_service.cc:1495] Processing CreateTablet for tablet 00dd2ffad1124244b171b2657d38a708 (DEFAULT_TABLE table=default.Table_e07b2ba4d9864d148d7088622b10e03c [id=201bf626c7d643d7a98a680b1e714ba3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.263150   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 1cf6d449fa2845a7b1f4cbd1de639d20 (table default.Table_650dc67e0ac74c149a3f64d456767938 [id=cfcd5fea97fb4ab3baff99eb28cd9476]) successfully deleted
I20261120 03:00:52.263355  1055 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00dd2ffad1124244b171b2657d38a708. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.263633  1387 raft_consensus.cc:534] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:52.264104  1387 raft_consensus.cc:556] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:52.237581  1384 raft_consensus.cc:556] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:52.266047  1387 leader_election.cc:290] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:52.267956   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3f65ba863447413cb122e728df3632fd" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:00:52.269358  1012 leader_election.cc:343] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: Tablet not RUNNING: STOPPED
I20261120 03:00:52.274726  1384 leader_election.cc:290] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:52.278228  1280 raft_consensus.cc:556] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:52.282809   500 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_fd084b0b1e49435c8bbbceca8642a022" } new_table_name: "default.Table_5c5911c8746d4b26843ec77c8141d3b6" modify_external_catalogs: true
I20261120 03:00:52.284729  1324 tablet_bootstrap.cc:654] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:52.287547  1054 tablet_service.cc:1542] Processing DeleteTablet for tablet df61317717d3401eb74cd93aa336c4dc with delete_type TABLET_DATA_DELETED (replaced by tablet 88b9e9692a874ae993dde4b9e2586a70) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:52.289343  1280 leader_election.cc:290] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:52.296348   755 tablet_service.cc:1495] Processing CreateTablet for tablet 020be7f7cda0467aab56f657cbe8d0ad (DEFAULT_TABLE table=default.Table_d1737969bdd644a6ab39da121eeedd60 [id=ed325a58ff124ba9a6b1d05bc901793b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.296674   757 tablet_service.cc:1495] Processing CreateTablet for tablet ce612f9a121b4b18908724d2726e156f (DEFAULT_TABLE table=default.Table_f601bffe6ddb49b3a3c589ac7b25235c [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.297490   755 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 020be7f7cda0467aab56f657cbe8d0ad. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.298614   757 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ce612f9a121b4b18908724d2726e156f. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.298846   756 tablet_service.cc:1495] Processing CreateTablet for tablet fbbe19dadb08455285078e239d81bcca (DEFAULT_TABLE table=default.Table_85ad53c3650a452b9c9ab5d515add49c [id=a8f19086139642fab5626a91bdaa6713]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.299870   756 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fbbe19dadb08455285078e239d81bcca. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.213538  1180 consensus_queue.cc:261] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:52.304577  1180 ts_tablet_manager.cc:1480] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.112s	user 0.005s	sys 0.000s
I20261120 03:00:52.309491  1054 tablet_service.cc:1542] Processing DeleteTablet for tablet 3f65ba863447413cb122e728df3632fd with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:51 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:52.313719  1054 tablet_service.cc:1542] Processing DeleteTablet for tablet 5f5c8711a6eb43c3aff1c27ed5115d6a with delete_type TABLET_DATA_DELETED (replaced by tablet a1854e4d9016414ea5c086ad56224b1f) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:52.321700   497 catalog_manager.cc:6146] Not sending DeleteTablet requests; no consensus state for tablet 251da439877644e68259e77d4d58d622
I20261120 03:00:52.330056   753 tablet_service.cc:1495] Processing CreateTablet for tablet daae644853734cd5818a46ecaa5ce7df (DEFAULT_TABLE table=default.Table_ae4914f59f3847c8a2a0ef1d68a7f4b2 [id=e540afdca1234f30bf1950448be502df]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.331226   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet daae644853734cd5818a46ecaa5ce7df. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.331360  1054 tablet_service.cc:1495] Processing CreateTablet for tablet 200b234da897419a82bf5844620adf24 (DEFAULT_TABLE table=default.Table_5303db2882934be1bfc2251ce51f2c8d [id=ae52d3d892e24bc38a8ff7d6db755c2d]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.332522  1054 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 200b234da897419a82bf5844620adf24. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.325708   499 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 5fee8dff291c43e9be0fb10b308718ce (table default.Table_a1f866996679431e9e1c4c5f6f8cdd12 [id=a275e215155b4824a1494f0954577fe8]). Expected version 1 got 0
I20261120 03:00:52.335381   496 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 0fd4a37075a34ec6bb0d6eb950045a1b
I20261120 03:00:52.312973   902 tablet_service.cc:1495] Processing CreateTablet for tablet ce612f9a121b4b18908724d2726e156f (DEFAULT_TABLE table=default.Table_f601bffe6ddb49b3a3c589ac7b25235c [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.319036   900 tablet_service.cc:1495] Processing CreateTablet for tablet daae644853734cd5818a46ecaa5ce7df (DEFAULT_TABLE table=default.Table_ae4914f59f3847c8a2a0ef1d68a7f4b2 [id=e540afdca1234f30bf1950448be502df]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.337069   900 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet daae644853734cd5818a46ecaa5ce7df. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.338941  1052 tablet_service.cc:1495] Processing CreateTablet for tablet 33e6cd23a3474034953c9b1a37b79acd (DEFAULT_TABLE table=default.Table_8c3f3e4b96ba410fa0109e46c25d4144 [id=9bbb6f66b4a841ac8bd1253946003db0]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.340055  1052 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 33e6cd23a3474034953c9b1a37b79acd. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.343325   901 tablet_service.cc:1542] Processing DeleteTablet for tablet 0fd4a37075a34ec6bb0d6eb950045a1b with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:51 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:52.344508   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ce612f9a121b4b18908724d2726e156f. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.345916   901 tablet_service.cc:1542] Processing DeleteTablet for tablet 9a31431495274c648625b75900b4ac47 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:52.345842  1053 tablet_service.cc:1495] Processing CreateTablet for tablet d27099bc667b4a1d880af7cec2715bb8 (DEFAULT_TABLE table=default.Table_d2f8156a63404ca5a92c6ea7190d15ad [id=767873bd6c694dcba7bc1df94dcf3925]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.347568  1053 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d27099bc667b4a1d880af7cec2715bb8. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.348613  1180 tablet_bootstrap.cc:492] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:52.355083  1051 tablet_service.cc:1495] Processing CreateTablet for tablet 9882c9cbfbd640a49e3e3c8904ffc1e1 (DEFAULT_TABLE table=default.Table_3ce972a9130f459f940b2910e9eb1427 [id=53fb1849755c4debad2dbfbf37271fbd]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.356336  1051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9882c9cbfbd640a49e3e3c8904ffc1e1. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.362506   901 tablet_service.cc:1542] Processing DeleteTablet for tablet df61317717d3401eb74cd93aa336c4dc with delete_type TABLET_DATA_DELETED (replaced by tablet 88b9e9692a874ae993dde4b9e2586a70) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:52.374831   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00dd2ffad1124244b171b2657d38a708" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:52.375587   923 raft_consensus.cc:2062] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:52.375882  1050 tablet_service.cc:1495] Processing CreateTablet for tablet 020be7f7cda0467aab56f657cbe8d0ad (DEFAULT_TABLE table=default.Table_d1737969bdd644a6ab39da121eeedd60 [id=ed325a58ff124ba9a6b1d05bc901793b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.376773  1046 tablet_service.cc:1542] Processing DeleteTablet for tablet 0fd4a37075a34ec6bb0d6eb950045a1b with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:51 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:52.377471  1050 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 020be7f7cda0467aab56f657cbe8d0ad. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.380079  1049 tablet_service.cc:1495] Processing CreateTablet for tablet fbbe19dadb08455285078e239d81bcca (DEFAULT_TABLE table=default.Table_85ad53c3650a452b9c9ab5d515add49c [id=a8f19086139642fab5626a91bdaa6713]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.381209  1049 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fbbe19dadb08455285078e239d81bcca. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.383477  1048 tablet_service.cc:1495] Processing CreateTablet for tablet ce612f9a121b4b18908724d2726e156f (DEFAULT_TABLE table=default.Table_f601bffe6ddb49b3a3c589ac7b25235c [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.385383  1047 tablet_service.cc:1495] Processing CreateTablet for tablet daae644853734cd5818a46ecaa5ce7df (DEFAULT_TABLE table=default.Table_ae4914f59f3847c8a2a0ef1d68a7f4b2 [id=e540afdca1234f30bf1950448be502df]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.388078  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0fd4a37075a34ec6bb0d6eb950045a1b" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:52.388839  1075 raft_consensus.cc:2062] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:52.388625   497 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_975b3ffda1cd474c8d82a00487c79adc"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:52.389861  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0fd4a37075a34ec6bb0d6eb950045a1b" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:52.390456  1074 raft_consensus.cc:3296] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
W20261120 03:00:52.390493   497 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_975b3ffda1cd474c8d82a00487c79adc in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:52.391353  1074 raft_consensus.cc:2062] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:52.398778  1180 tablet_bootstrap.cc:654] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:52.399295   493 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_3ce972a9130f459f940b2910e9eb1427" } new_table_name: "default.Table_0879bd445c234e4faefff97cd088035d" modify_external_catalogs: true
I20261120 03:00:52.409545   901 tablet_service.cc:1495] Processing CreateTablet for tablet 68179488b9bc4e03b65a9f1d434c95b5 (DEFAULT_TABLE table=default.Table_650dc67e0ac74c149a3f64d456767938 [id=cfcd5fea97fb4ab3baff99eb28cd9476]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.410665   901 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 68179488b9bc4e03b65a9f1d434c95b5. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.414855   494 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_8fdb321b36394e5ea7df159be584695b"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:52.416666   494 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_8fdb321b36394e5ea7df159be584695b in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:52.420573   752 tablet_service.cc:1542] Processing DeleteTablet for tablet 0fd4a37075a34ec6bb0d6eb950045a1b with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:51 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:52.422556   499 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_7f80bbdf810749109360575194927d04"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:52.426123  1185 tablet_replica.cc:333] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:52.401732   900 tablet_service.cc:1495] Processing CreateTablet for tablet 1885397e42774a89bb347bc776284bc0 (DEFAULT_TABLE table=default.Table_bc7d30a65be54db09b1378e3b21551f4 [id=b5b565c2c2224675baada2ae9b42be54]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.432685   900 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1885397e42774a89bb347bc776284bc0. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.435202   752 tablet_service.cc:1542] Processing DeleteTablet for tablet 3f65ba863447413cb122e728df3632fd with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:51 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:52.433621  1295 ts_tablet_manager.cc:1975] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:52.440233  1295 log.cc:1199] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/3f65ba863447413cb122e728df3632fd
I20261120 03:00:52.440827   750 tablet_service.cc:1542] Processing DeleteTablet for tablet 9a31431495274c648625b75900b4ac47 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:52.441805  1295 ts_tablet_manager.cc:1996] T 3f65ba863447413cb122e728df3632fd P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:52.445669   710 leader_election.cc:304] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:52.445988   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 3f65ba863447413cb122e728df3632fd (table default.Table_4755bb4a11ae4e33bff2a6cc7384921a [id=676ae8fc1a8c488d820a11b930b0ed5f]) successfully deleted
I20261120 03:00:52.447517   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9a31431495274c648625b75900b4ac47" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:52.435328   496 catalog_manager.cc:5856] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65). New cstate: current_term: 1 leader_uuid: "5beeefe8bef54caeac463d05d8c30fa6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } }
W20261120 03:00:52.447698   499 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_7f80bbdf810749109360575194927d04 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:52.451778  1355 leader_election.cc:290] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:52.452323  1355 raft_consensus.cc:3024] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:52.452620  1355 raft_consensus.cc:534] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:52.452901  1355 raft_consensus.cc:3296] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:52.454478   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d27099bc667b4a1d880af7cec2715bb8" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
W20261120 03:00:52.456276   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet c3b56d0ccab140c28a37808d9d5902c6 because the tablet was not found. No further retry: Not found: Tablet not found: c3b56d0ccab140c28a37808d9d5902c6
W20261120 03:00:52.460716   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet a642c32c1b6043978516d97efa2347a3 because the tablet was not found. No further retry: Not found: Tablet not found: a642c32c1b6043978516d97efa2347a3
I20261120 03:00:52.460880  1185 raft_consensus.cc:2440] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:52.461635  1185 raft_consensus.cc:2469] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:52.463788  1185 ts_tablet_manager.cc:1962] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:52.469229   750 tablet_service.cc:1495] Processing CreateTablet for tablet 68179488b9bc4e03b65a9f1d434c95b5 (DEFAULT_TABLE table=default.Table_650dc67e0ac74c149a3f64d456767938 [id=cfcd5fea97fb4ab3baff99eb28cd9476]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.449680   776 raft_consensus.cc:3296] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:52.470908   776 raft_consensus.cc:2062] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:00:52.466984   496 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 5fee8dff291c43e9be0fb10b308718ce (table default.Table_a1f866996679431e9e1c4c5f6f8cdd12 [id=a275e215155b4824a1494f0954577fe8]). Expected version 1 got 0
W20261120 03:00:52.475340   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 06c3bac919e342b09dd6af13aebc387b because the tablet was not found. No further retry: Not found: Tablet not found: 06c3bac919e342b09dd6af13aebc387b
I20261120 03:00:52.476838  1295 tablet_replica.cc:333] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:52.478123  1295 raft_consensus.cc:2440] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:52.480847  1295 raft_consensus.cc:2469] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Raft consensus is shut down!
W20261120 03:00:52.481182   857 leader_election.cc:343] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:52.482270   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a1f6197c60dd4dfb8aeb5c38373aab21" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:52.483822   923 raft_consensus.cc:3296] T a1f6197c60dd4dfb8aeb5c38373aab21 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:52.484859   923 raft_consensus.cc:2062] T a1f6197c60dd4dfb8aeb5c38373aab21 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:52.484411  1046 tablet_service.cc:1495] Processing CreateTablet for tablet 1885397e42774a89bb347bc776284bc0 (DEFAULT_TABLE table=default.Table_bc7d30a65be54db09b1378e3b21551f4 [id=b5b565c2c2224675baada2ae9b42be54]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.487171  1076 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "fa57d0c282964e37b9133ea57f458a51" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:52.488881  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "fa57d0c282964e37b9133ea57f458a51" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:52.490952  1295 ts_tablet_manager.cc:1962] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:52.491369  1077 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5fee8dff291c43e9be0fb10b308718ce" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:52.453584  1355 raft_consensus.cc:556] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:52.498636   500 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 5c5c0ac869a1419fad9f61ec647027ee has no consensus state
I20261120 03:00:52.493743  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5fee8dff291c43e9be0fb10b308718ce" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:52.501236   495 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet fa57d0c282964e37b9133ea57f458a51
I20261120 03:00:52.502789   495 catalog_manager.cc:6952] ReplaceTablet: tablet fa57d0c282964e37b9133ea57f458a51 deleted and replaced by tablet c6a65c1a4a574961bec657ad7c9048e0
I20261120 03:00:52.505427   903 tablet_service.cc:1542] Processing DeleteTablet for tablet fa57d0c282964e37b9133ea57f458a51 with delete_type TABLET_DATA_DELETED (replaced by tablet c6a65c1a4a574961bec657ad7c9048e0) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:52.506850  1355 leader_election.cc:290] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
W20261120 03:00:52.526443   489 catalog_manager.cc:4863] Async tablet task AlterTable RPC for tablet 5c5c0ac869a1419fad9f61ec647027ee (table default.Table_5c5911c8746d4b26843ec77c8141d3b6 [id=ffb2040a2e114893855d4d8c66dfa855], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 5c5c0ac869a1419fad9f61ec647027ee has no consensus state
W20261120 03:00:52.527503   493 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 9882c9cbfbd640a49e3e3c8904ffc1e1 consensus state has no leader
I20261120 03:00:52.496212   710 leader_election.cc:304] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:52.533134   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00dd2ffad1124244b171b2657d38a708" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
W20261120 03:00:52.504212   712 consensus_peers.cc:183] SignalRequest failed: Illegal state: peer closed
I20261120 03:00:52.533710   923 raft_consensus.cc:3296] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:52.534627   923 raft_consensus.cc:2062] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:52.506989   750 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 68179488b9bc4e03b65a9f1d434c95b5. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.535625   757 tablet_service.cc:1542] Processing DeleteTablet for tablet fa57d0c282964e37b9133ea57f458a51 with delete_type TABLET_DATA_DELETED (replaced by tablet c6a65c1a4a574961bec657ad7c9048e0) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:52.536787  1355 raft_consensus.cc:3024] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:52.537230  1355 raft_consensus.cc:744] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:52.537987  1355 consensus_queue.cc:238] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:52.541960   493 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_863c1a82fb1a4f2db6a422422dc64c04" } modify_external_catalogs: true
I20261120 03:00:52.542500   493 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_863c1a82fb1a4f2db6a422422dc64c04" } modify_external_catalogs: true
I20261120 03:00:52.545919   497 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_e07b2ba4d9864d148d7088622b10e03c" } modify_external_catalogs: true
I20261120 03:00:52.546465   497 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_e07b2ba4d9864d148d7088622b10e03c" } modify_external_catalogs: true
I20261120 03:00:52.425760  1289 raft_consensus.cc:556] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:52.550935   494 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_027fdfc1742a404ea34dfdc00e41ff5a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:52.552700   494 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_027fdfc1742a404ea34dfdc00e41ff5a in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:52.507532  1185 ts_tablet_manager.cc:1975] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:52.554303  1185 log.cc:1199] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/0fd4a37075a34ec6bb0d6eb950045a1b
I20261120 03:00:52.555059  1185 ts_tablet_manager.cc:1996] T 0fd4a37075a34ec6bb0d6eb950045a1b P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:52.494822   751 tablet_service.cc:1495] Processing CreateTablet for tablet 1885397e42774a89bb347bc776284bc0 (DEFAULT_TABLE table=default.Table_bc7d30a65be54db09b1378e3b21551f4 [id=b5b565c2c2224675baada2ae9b42be54]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.555972   751 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1885397e42774a89bb347bc776284bc0. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.564606  1185 tablet_replica.cc:333] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:52.565344  1185 raft_consensus.cc:2440] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:52.565814  1185 raft_consensus.cc:2469] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:52.567519  1185 ts_tablet_manager.cc:1962] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:52.568342   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 0fd4a37075a34ec6bb0d6eb950045a1b (table default.Table_d52207672f8f40acb7026fec3eb73d85 [id=563f250386d546eb8b53db93e8139e9b]) successfully deleted
W20261120 03:00:52.569080   486 catalog_manager.cc:5048] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 3f65ba863447413cb122e728df3632fd because the tablet was not found. No further retry: Not found: Tablet not found: 3f65ba863447413cb122e728df3632fd
W20261120 03:00:52.571131   860 consensus_peers.cc:183] SignalRequest failed: Illegal state: peer closed
I20261120 03:00:52.571266   710 leader_election.cc:304] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:52.581300  1389 raft_consensus.cc:3024] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:52.583766   495 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 5fee8dff291c43e9be0fb10b308718ce (table default.Table_a1f866996679431e9e1c4c5f6f8cdd12 [id=a275e215155b4824a1494f0954577fe8]). Expected version 1 got 0
I20261120 03:00:52.589332  1389 raft_consensus.cc:744] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:52.590322  1389 consensus_queue.cc:238] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:52.601608  1289 leader_election.cc:290] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:52.602284   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "4abb67f3819a45b9b05dde04842e2c9d" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:52.602860   923 raft_consensus.cc:3296] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:52.603736   923 raft_consensus.cc:2062] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
==================
WARNING: ThreadSanitizer: data race (pid=977)
  Write of size 8 at 0x7b44000b4570 by thread T127:
I20261120 03:00:52.611820  1044 tablet_service.cc:1542] Processing DeleteTablet for tablet fa57d0c282964e37b9133ea57f458a51 with delete_type TABLET_DATA_DELETED (replaced by tablet c6a65c1a4a574961bec657ad7c9048e0) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:52.612568  1045 tablet_service.cc:1495] Processing CreateTablet for tablet 68179488b9bc4e03b65a9f1d434c95b5 (DEFAULT_TABLE table=default.Table_650dc67e0ac74c149a3f64d456767938 [id=cfcd5fea97fb4ab3baff99eb28cd9476]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:52.618956  1289 consensus_queue.cc:1049] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:52.622421   500 master_service.cc:965] ReplaceTablet: received request to replace tablet fa57d0c282964e37b9133ea57f458a51 from {username='slave'} at 127.0.0.1:45640
I20261120 03:00:52.627468   857 leader_election.cc:304] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:52.627823  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00dd2ffad1124244b171b2657d38a708" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:52.628221  1380 raft_consensus.cc:3024] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:52.628666  1380 raft_consensus.cc:744] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:00:52.629410  1380 consensus_queue.cc:238] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:52.631420  1180 tablet_bootstrap.cc:492] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:52.631793  1180 ts_tablet_manager.cc:1449] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.283s	user 0.010s	sys 0.001s
    #0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (kudu+0x321819)
    #1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (kudu+0x3287c9)
    #2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (kudu+0x328769)
    #3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x2aebd2)
    #4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x2aeb69)
    #5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x381bcd)
    #6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x381500)
    #7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
    #8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libkudu_cli_actions.so+0x24d1d5)
    #9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #10 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x37a1db)
    #11 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x381495)
    #12 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libkudu_cli_actions.so+0x24d225)
    #13 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libkudu_cli_actions.so+0x24d1c9)
    #14 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #15 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x450864)
    #16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x403c45)
    #17 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x403e59)
    #18 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libkudu_cli_actions.so+0x2fbea7)
    #19 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libkudu_cli_actions.so+0x2fbe69)
    #20 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libkudu_cli_actions.so+0x2fbe29)
    #21 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libkudu_cli_actions.so+0x2fa1fa)
    #22 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3e1f09)
    #23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3e1ee0)
    #24 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3e1ea0)
    #25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b4aa6)
    #26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b49c4)
    #27 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b477b)
    #28 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x28c761)
    #29 kudu::tserver::TSTabletManager::PopulateIncrementalTabletReport(kudu::master::TabletReportPB*, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, bool) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1893:1 (libtserver.so+0x2c1b50)
    #30 kudu::tserver::Heartbeater::Thread::GenerateIncrementalTabletReport(kudu::master::TabletReportPB*, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:790:30 (libtserver.so+0x1cb58a)
    #31 kudu::tserver::Heartbeater::Thread::DoHeartbeat(kudu::master::MasterErrorPB*, kudu::rpc::ErrorStatusPB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:521:5 (libtserver.so+0x1cd4c4)
    #32 kudu::tserver::Heartbeater::Thread::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:645:21 (libtserver.so+0x1ce790)
    #33 kudu::tserver::Heartbeater::Thread::Start()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:728:48 (libtserver.so+0x1d03c1)
    #34 decltype(std::__1::forward<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(fp)()) std::__1::__invoke<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x1d0379)
    #35 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x1d0309)
    #36 std::__1::__function::__alloc_func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x1d02d1)
    #37 std::__1::__function::__func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x1cf5cd)
    #38 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #39 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #40 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Previous atomic write of size 8 at 0x7b44000b4570 by thread T144 (mutexes: write M299906313465758112):
    #0 __tsan_atomic64_fetch_add /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:620 (kudu+0x2ca8f5)
    #1 long std::__1::__cxx_atomic_fetch_add<long>(std::__1::__cxx_atomic_base_impl<long>*, long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1042:12 (libkudu_util.so+0x2c2cd3)
    #2 std::__1::__atomic_base<long, true>::fetch_add(long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1706:17 (libkudu_util.so+0x2c2c09)
    #3 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:30 (libkudu_util.so+0x37b5f2)
    #4 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x378c61)
    #5 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x451230)
    #6 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:608:16 (libtablet.so+0x44a1c2)
    #7 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:579:5 (libtablet.so+0x449953)
    #8 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:65 (libtablet.so+0x44db61)
    #9 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x44db19)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x44daa9)
    #11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x44da71)
    #12 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x44cc9d)
    #13 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #14 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #15 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x4403a5)
    #16 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x442711)
I20261120 03:00:52.638913  1324 tablet_bootstrap.cc:492] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:52.639336  1324 ts_tablet_manager.cc:1449] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.407s	user 0.011s	sys 0.007s
    #17 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x4426c9)
    #18 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x442659)
    #19 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x442621)
    #20 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x44191d)
    #21 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #22 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #23 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Mutex M299906313465758112 is already destroyed.

  Thread T127 'heartbeat-1124' (tid=1124, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::tserver::Heartbeater::Thread::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:727:10 (libtserver.so+0x1cae61)
I20261120 03:00:52.649816  1400 raft_consensus.cc:534] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
    #4 kudu::tserver::Heartbeater::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:251:41 (libtserver.so+0x1cac3d)
    #5 kudu::tserver::TabletServer::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:167:3 (libtserver.so+0x259ccc)
    #6 kudu::tserver::RunTabletServer() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server_runner.cc:86:3 (libtserver.so+0x266e93)
I20261120 03:00:52.650552  1400 raft_consensus.cc:556] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
    #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:128:10 (libkudu_cli_actions.so+0x3b6429)
    #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_cli_actions.so+0x28c051)
    #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (libkudu_cli_actions.so+0x28bfbc)
    #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_cli_actions.so+0x28bf2c)
    #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_cli_actions.so+0x28ab95)
    #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x14082f)
    #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132da1)
    #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:270:10 (libkudu_tools_util.so+0x130a52)
    #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:114:15 (kudu+0x324910)
    #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:186:16 (kudu+0x325247)
    #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:249:10 (kudu+0x325671)

  Thread T144 'apply [worker]-' (tid=1286, finished) created by thread T82 at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x43f5cb)
    #4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x43d8cb)
    #5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x43fb32)
    #6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:23 (libtablet.so+0x448cf0)
    #7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:476:5 (libtablet.so+0x449289)
    #8 kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:860:15 (libtablet.so+0x41362a)
    #9 decltype(std::__1::forward<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x4135b8)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x413538)
    #11 std::__1::__function::__alloc_func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x4134f3)
    #12 std::__1::__function::__func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41270c)
W20261120 03:00:52.661569   709 leader_election.cc:343] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
    #13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x40fe06)
    #14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x40a5b8)
    #15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3598:3 (libconsensus.so+0x1fd9c0)
    #16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:200:12 (libconsensus.so+0x1d2e9d)
    #17 kudu::consensus::RaftConsensus::UpdateReplica(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::consensus::RaftConsensus::FlushContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1603:5 (libconsensus.so+0x1f0713)
    #18 kudu::consensus::RaftConsensus::Update(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1162:9 (libconsensus.so+0x1efd04)
I20261120 03:00:52.664889  1400 leader_election.cc:290] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:52.665447   493 catalog_manager.cc:6146] Not sending DeleteTablet requests; no consensus state for tablet 5144400be7ee47e783e1266eac8ded62
    #19 kudu::tserver::ConsensusServiceImpl::UpdateConsensus(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1792:25 (libtserver.so+0x26e199)
    #20 kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/consensus/consensus.service.cc:321:13 (libconsensus_proto.so+0xd8d34)
    #21 decltype(std::__1::forward<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus_proto.so+0xd8cc2)
    #22 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus_proto.so+0xd8bf1)
    #23 std::__1::__function::__alloc_func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus_proto.so+0xd8b6c)
I20261120 03:00:52.669150  1180 raft_consensus.cc:366] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
    #24 std::__1::__function::__func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus_proto.so+0xd7e22)
    #25 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x21b0bc)
    #26 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x21a4f6)
I20261120 03:00:52.641776  1324 raft_consensus.cc:366] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:52.673517  1324 raft_consensus.cc:392] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:52.673781  1324 raft_consensus.cc:787] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
    #27 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x219e9f)
    #28 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15 (libkrpc.so+0x21d16f)
    #29 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5 (libkrpc.so+0x21e4c1)
    #30 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x21e479)
    #31 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x21e409)
    #32 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x21e3d1)
    #33 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x21d6cd)
    #34 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #35 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #36 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
==================
I20261120 03:00:52.674875   496 catalog_manager.cc:6409] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: default.Table_523caa128be441e1a52767e3d3318ce0 [id=93a0027d82064a208e2535349e9b042d] alter complete (version 1)
I20261120 03:00:52.677630  1382 tablet.cc:1872] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20261120 03:00:52.683275  1048 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ce612f9a121b4b18908724d2726e156f. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:00:52.690856   710 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 46062us
I20261120 03:00:52.702543   494 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_e63a7bef412545baac9d0cae0cf299ea" } new_table_name: "default.Table_e4c5ff8f28ae4efca67257942a4fcb77" modify_external_catalogs: true
I20261120 03:00:52.709391  1289 consensus_queue.cc:1049] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20261120 03:00:52.712829  1078 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.96.66:60421 (request call id 16) took 1939 ms. Trace:
I20261120 03:00:52.713377  1078 rpcz_store.cc:276] 1120 03:00:52.192056 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:52.192191 (+   135us) service_pool.cc:224] Handling call
1120 03:00:52.192510 (+   319us) raft_consensus.cc:1536] Updating replica for 1 ops
1120 03:00:52.244137 (+ 51627us) raft_consensus.cc:1602] Early marking committed up to index 0
1120 03:00:52.244171 (+    34us) raft_consensus.cc:1607] Triggering prepare for 1 ops
1120 03:00:52.244624 (+   453us) log.cc:844] Serialized 26 byte log entry
1120 03:00:52.712471 (+467847us) raft_consensus.cc:1713] Marking committed up to 0
1120 03:00:52.712512 (+    41us) raft_consensus.cc:1765] Filling consensus response to leader.
1120 03:00:52.712588 (+    76us) raft_consensus.cc:1736] Waiting on the replicates to finish logging
1120 03:00:52.712632 (+    44us) raft_consensus.cc:1753] finished
1120 03:00:52.712645 (+    13us) raft_consensus.cc:1759] UpdateReplicas() finished
1120 03:00:52.712769 (+   124us) inbound_call.cc:177] Queueing success response
Metrics: {"raft.queue_time_us":82553,"raft.run_cpu_time_us":124,"raft.run_wall_time_us":125,"thread_start_us":518567,"threads_started":2,"wal-append.queue_time_us":442602}
I20261120 03:00:52.732383   495 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_bc7d30a65be54db09b1378e3b21551f4" } new_table_name: "default.Table_61cb1e03b7e442a8822e32a2295866f3" modify_external_catalogs: true
I20261120 03:00:52.733326   710 leader_election.cc:304] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:52.736455   493 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_555e4ecfadce486baefa8c3e2d1bc272"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:52.737609  1289 raft_consensus.cc:3024] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
W20261120 03:00:52.738539   493 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_555e4ecfadce486baefa8c3e2d1bc272 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:52.738626  1289 raft_consensus.cc:744] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:52.739816  1289 consensus_queue.cc:238] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:52.747730  1185 ts_tablet_manager.cc:1975] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
W20261120 03:00:52.750180  1181 thread.cc:642] raft [worker] (thread pool) Time spent creating pthread: real 0.504s	user 0.005s	sys 0.021s
W20261120 03:00:52.750478  1181 thread.cc:609] raft [worker] (thread pool) Time spent starting thread: real 0.505s	user 0.005s	sys 0.021s
I20261120 03:00:52.751044   499 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_cfa8989fb1134d908ab28bd6ebb80563"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:52.751669  1181 ts_tablet_manager.cc:1480] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.511s	user 0.009s	sys 0.021s
W20261120 03:00:52.751931  1181 ts_tablet_manager.cc:1513] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234: Tablet startup took 511ms
W20261120 03:00:52.752533   499 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_cfa8989fb1134d908ab28bd6ebb80563 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:00:52.752149  1181 ts_tablet_manager.cc:1515] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234: Trace:
1120 03:00:50.536453 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:50.536560 (+   107us) service_pool.cc:224] Handling call
1120 03:00:50.537708 (+  1148us) ts_tablet_manager.cc:715] Acquired tablet manager lock
1120 03:00:50.537781 (+    73us) ts_tablet_manager.cc:728] Creating new metadata...
1120 03:00:50.545083 (+  7302us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:50.553313 (+  8230us) tablet_replica.cc:206] Creating consensus instance
1120 03:00:50.578127 (+ 24814us) inbound_call.cc:177] Queueing success response
1120 03:00:50.640731 (+ 62604us) ts_tablet_manager.cc:1425] Bootstrapping tablet
1120 03:00:52.240489 (+1599758us) ts_tablet_manager.cc:1481] Starting tablet replica
1120 03:00:52.241020 (+   531us) tablet_replica.cc:261] Starting instrumentation
1120 03:00:52.241477 (+   457us) tablet_replica.cc:280] Starting consensus
Metrics: {"raft.queue_time_us":29766,"raft.run_cpu_time_us":148,"raft.run_wall_time_us":147,"tablet-open.queue_time_us":62711,"thread_start_us":504819,"threads_started":1}
I20261120 03:00:52.753760  1181 tablet_bootstrap.cc:492] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
W20261120 03:00:52.754146  1011 thread.cc:642] client-negotiator [worker] (thread pool) Time spent creating pthread: real 0.561s	user 0.004s	sys 0.027s
W20261120 03:00:52.754426  1011 thread.cc:609] client-negotiator [worker] (thread pool) Time spent starting thread: real 0.561s	user 0.004s	sys 0.027s
W20261120 03:00:52.758136  1079 thread.cc:642] apply [worker] (thread pool) Time spent creating pthread: real 0.567s	user 0.011s	sys 0.020s
W20261120 03:00:52.758438  1079 thread.cc:609] apply [worker] (thread pool) Time spent starting thread: real 0.567s	user 0.011s	sys 0.020s
W20261120 03:00:52.758651  1012 consensus_peers.cc:597] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 -> Peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Couldn't send request to peer 3d30c2428730481db9b0ffd279dd3440. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a642c32c1b6043978516d97efa2347a3. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:52.759083  1079 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.96.65:39215 (request call id 29) took 1982 ms. Trace:
I20261120 03:00:52.759439  1079 rpcz_store.cc:276] 1120 03:00:52.188691 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:52.190428 (+  1737us) service_pool.cc:224] Handling call
1120 03:00:52.190730 (+   302us) raft_consensus.cc:1536] Updating replica for 0 ops
1120 03:00:52.190904 (+   174us) raft_consensus.cc:1602] Early marking committed up to index 2
1120 03:00:52.758745 (+567841us) raft_consensus.cc:1607] Triggering prepare for 0 ops
1120 03:00:52.758801 (+    56us) raft_consensus.cc:1713] Marking committed up to 2
1120 03:00:52.758844 (+    43us) raft_consensus.cc:1765] Filling consensus response to leader.
1120 03:00:52.758911 (+    67us) raft_consensus.cc:1759] UpdateReplicas() finished
1120 03:00:52.759027 (+   116us) inbound_call.cc:177] Queueing success response
Metrics: {}
I20261120 03:00:52.670060  1180 raft_consensus.cc:392] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:52.693190   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5a3fe1499d0b4fab9f9949c8ded0a8dc" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:52.761529   924 raft_consensus.cc:2062] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
W20261120 03:00:52.762852  1012 consensus_peers.cc:597] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 -> Peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Couldn't send request to peer 3d30c2428730481db9b0ffd279dd3440. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d753dd170cab4f18980e85707c4093e9. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:52.732097  1295 ts_tablet_manager.cc:1975] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:52.763851  1295 log.cc:1199] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/136056e5242a42ef8fa4b40c989b5423
W20261120 03:00:52.744503   857 consensus_peers.cc:597] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 -> Peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Couldn't send request to peer 5beeefe8bef54caeac463d05d8c30fa6. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:52.769418  1295 ts_tablet_manager.cc:1996] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:52.771420   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 136056e5242a42ef8fa4b40c989b5423 (table default.Table_d2ca6a35f8b842d784167fd22339644a [id=afefca6af8d64812b4f3e37317a1c8d7]) successfully deleted
W20261120 03:00:52.772279   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet a72d5d7896de4020a1ac4d2f8c551e15 because the tablet was not found. No further retry: Not found: Tablet not found: a72d5d7896de4020a1ac4d2f8c551e15
I20261120 03:00:52.772418  1295 tablet_replica.cc:333] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:52.773119  1295 raft_consensus.cc:2440] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:52.773628  1295 raft_consensus.cc:2469] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:52.775517  1295 ts_tablet_manager.cc:1962] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:52.776813  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3f65ba863447413cb122e728df3632fd" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:52.777029  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3f65ba863447413cb122e728df3632fd" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:52.760921  1180 raft_consensus.cc:787] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:52.778872  1069 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "4abb67f3819a45b9b05dde04842e2c9d" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:52.781025  1072 raft_consensus.cc:3296] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:52.777618  1078 raft_consensus.cc:2062] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:52.782919  1180 consensus_queue.cc:261] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:52.789932  1180 ts_tablet_manager.cc:1480] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.158s	user 0.007s	sys 0.000s
I20261120 03:00:52.793219  1185 log.cc:1199] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/9a31431495274c648625b75900b4ac47
I20261120 03:00:52.793903  1185 ts_tablet_manager.cc:1996] T 9a31431495274c648625b75900b4ac47 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:52.794458  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "df61317717d3401eb74cd93aa336c4dc" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:52.793808  1069 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5fee8dff291c43e9be0fb10b308718ce" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:52.795975  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "136056e5242a42ef8fa4b40c989b5423" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:52.795993  1071 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5a3fe1499d0b4fab9f9949c8ded0a8dc" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:52.796816  1078 raft_consensus.cc:2628] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 5beeefe8bef54caeac463d05d8c30fa6 in this term.
I20261120 03:00:52.797508  1180 tablet_bootstrap.cc:492] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:52.798136  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9a31431495274c648625b75900b4ac47" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:52.799980  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d27099bc667b4a1d880af7cec2715bb8" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:52.801633  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9a31431495274c648625b75900b4ac47" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
W20261120 03:00:52.769686   709 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 107974us
I20261120 03:00:52.770049  1324 consensus_queue.cc:261] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
W20261120 03:00:52.802502   709 leader_election.cc:343] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:52.803449   486 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111) [suppressed 10 similar messages]
I20261120 03:00:52.808557  1079 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1237b0d2fa414ef38271835b5d69d6b9" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:52.813416   857 leader_election.cc:343] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:52.813823   857 leader_election.cc:304] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
W20261120 03:00:52.814324   709 consensus_peers.cc:597] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
W20261120 03:00:52.814877   709 leader_election.cc:343] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:52.815289  1280 raft_consensus.cc:2966] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20261120 03:00:52.815640   709 leader_election.cc:343] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Not found: Tablet not found: 4abb67f3819a45b9b05dde04842e2c9d
W20261120 03:00:52.816541   494 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 88b9e9692a874ae993dde4b9e2586a70 has no consensus state
W20261120 03:00:52.819806   857 leader_election.cc:343] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:52.820180   857 leader_election.cc:304] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:52.816838  1400 consensus_queue.cc:1049] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.004s
I20261120 03:00:52.816723  1353 raft_consensus.cc:534] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
W20261120 03:00:52.824739   857 raft_consensus.cc:2902] T df61317717d3401eb74cd93aa336c4dc P 3d30c2428730481db9b0ffd279dd3440: unable to run election callback: Service unavailable: Thread pool token was shut down
I20261120 03:00:52.823843  1353 raft_consensus.cc:556] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:52.828860  1324 ts_tablet_manager.cc:1480] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.189s	user 0.007s	sys 0.000s
I20261120 03:00:52.829773  1324 tablet_bootstrap.cc:492] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:52.832047   710 leader_election.cc:304] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
W20261120 03:00:52.833839   711 consensus_peers.cc:183] SignalRequest failed: Illegal state: peer closed
W20261120 03:00:52.838327   857 leader_election.cc:343] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:52.838752   857 leader_election.cc:304] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:52.839457  1280 raft_consensus.cc:2966] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:52.840730  1181 tablet_bootstrap.cc:654] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:52.843365   857 leader_election.cc:343] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:52.843752   857 leader_election.cc:304] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
W20261120 03:00:52.844175   495 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 1885397e42774a89bb347bc776284bc0 consensus state has no leader
I20261120 03:00:52.844394  1280 raft_consensus.cc:2966] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:52.846107   857 leader_election.cc:304] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
W20261120 03:00:52.846664   857 raft_consensus.cc:2902] T 136056e5242a42ef8fa4b40c989b5423 P 3d30c2428730481db9b0ffd279dd3440: unable to run election callback: Service unavailable: Thread pool token was shut down
W20261120 03:00:52.852003   709 leader_election.cc:343] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Not found: Tablet not found: a1f6197c60dd4dfb8aeb5c38373aab21
W20261120 03:00:52.852511  1011 consensus_peers.cc:597] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 -> Peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Couldn't send request to peer 5beeefe8bef54caeac463d05d8c30fa6. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d753dd170cab4f18980e85707c4093e9. This is attempt 1: this message will repeat every 5th retry.
W20261120 03:00:52.852885   709 leader_election.cc:343] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Not found: Tablet not found: 00dd2ffad1124244b171b2657d38a708
W20261120 03:00:52.853173   857 leader_election.cc:343] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:52.854063   857 leader_election.cc:343] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:52.854794   857 leader_election.cc:304] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:52.854962  1400 raft_consensus.cc:3024] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:52.855348  1400 raft_consensus.cc:534] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:52.855684  1400 raft_consensus.cc:3296] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:52.857654  1353 leader_election.cc:290] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:52.859042  1280 raft_consensus.cc:2966] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20261120 03:00:52.860056   857 leader_election.cc:343] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:52.860844   857 consensus_peers.cc:597] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:52.862025   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "200b234da897419a82bf5844620adf24" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:00:52.862463  1011 leader_election.cc:343] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: 3f65ba863447413cb122e728df3632fd
I20261120 03:00:52.862776   923 raft_consensus.cc:2062] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:52.862884  1011 leader_election.cc:304] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:52.864507   710 leader_election.cc:304] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:52.865736  1289 raft_consensus.cc:3024] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:52.866099  1289 raft_consensus.cc:534] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:52.866441  1289 raft_consensus.cc:3296] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:52.867173  1289 raft_consensus.cc:556] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:52.868840  1391 raft_consensus.cc:2966] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:52.878538   857 leader_election.cc:304] T 1237b0d2fa414ef38271835b5d69d6b9 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:52.879230  1280 raft_consensus.cc:2966] T 1237b0d2fa414ef38271835b5d69d6b9 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20261120 03:00:52.889658   709 consensus_peers.cc:597] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:52.876933  1300 master_runner.cc:387] Master server non-default flags:
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/logs
--logbuflevel=-1
--logtostderr=true
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.96.84:35145
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/data
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/wal
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.96.126:44467,127.0.96.125:37411,127.0.96.124:38133
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.124:38133
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/data/info.pb
--webserver_interface=127.0.96.124
--webserver_port=43247
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false

Master server version:
kudu 1.19.0-SNAPSHOT
revision 36b6772d828145f8fbba3c49746c86838683f9ba
build type FASTDEBUG
built by None at 20 Nov 2026 02:45:53 UTC on bdcb31816ec0
build id 14870
TSAN enabled
I20261120 03:00:52.900002  1300 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20261120 03:00:52.911005   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 9a31431495274c648625b75900b4ac47 (table default.Table_3bb077af5e15449b85500aca3a1500c3 [id=234b77d1cf1847b0ae8afa8d3dbdff71]) successfully deleted
I20261120 03:00:52.914383  1300 file_cache.cc:504] Constructed file cache file cache with capacity 419430
W20261120 03:00:52.919201  1135 outbound_call.cc:321] RPC callback for RPC call kudu.master.MasterService.AlterTable -> {remote=127.0.96.125:37411, user_credentials={real_user=slave}} blocked reactor thread for 99939.8us
I20261120 03:00:52.923656   495 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_3c9436f0bbcd41bb987053b52acc02cf"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:52.925482   495 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_3c9436f0bbcd41bb987053b52acc02cf in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:52.926507   493 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_5303db2882934be1bfc2251ce51f2c8d" } modify_external_catalogs: true
I20261120 03:00:52.927066   493 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_5303db2882934be1bfc2251ce51f2c8d" } modify_external_catalogs: true
I20261120 03:00:52.929277   494 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_646d629dfeea413a86bc3edf68c7d6ae"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:52.931026   494 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_646d629dfeea413a86bc3edf68c7d6ae in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:52.936447   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "200b234da897419a82bf5844620adf24" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:52.937098   924 raft_consensus.cc:3296] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:52.942250   924 raft_consensus.cc:2062] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
==================
WARNING: ThreadSanitizer: data race (pid=977)
  Write of size 8 at 0x7b44000b4578 by thread T127:
    #0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (kudu+0x321819)
    #1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (kudu+0x3287c9)
    #2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (kudu+0x328769)
    #3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x2aebd2)
    #4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x2aeb69)
    #5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x381bcd)
    #6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x381500)
    #7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
    #8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libkudu_cli_actions.so+0x24d1d5)
    #9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #10 kudu::MemTracker::~MemTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:83:1 (libkudu_util.so+0x37a1db)
    #11 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3503:23 (libkudu_util.so+0x381495)
    #12 std::__1::__shared_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3341:9 (libkudu_cli_actions.so+0x24d225)
    #13 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3383:27 (libkudu_cli_actions.so+0x24d1c9)
    #14 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #15 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x450864)
    #16 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x403c45)
    #17 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x403e59)
    #18 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libkudu_cli_actions.so+0x2fbea7)
    #19 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libkudu_cli_actions.so+0x2fbe69)
    #20 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libkudu_cli_actions.so+0x2fbe29)
    #21 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libkudu_cli_actions.so+0x2fa1fa)
    #22 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3e1f09)
    #23 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3e1ee0)
    #24 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3e1ea0)
    #25 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b4aa6)
    #26 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b49c4)
    #27 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b477b)
    #28 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x28c761)
    #29 kudu::tserver::TSTabletManager::PopulateIncrementalTabletReport(kudu::master::TabletReportPB*, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, bool) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1893:1 (libtserver.so+0x2c1b50)
    #30 kudu::tserver::Heartbeater::Thread::GenerateIncrementalTabletReport(kudu::master::TabletReportPB*, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:790:30 (libtserver.so+0x1cb58a)
    #31 kudu::tserver::Heartbeater::Thread::DoHeartbeat(kudu::master::MasterErrorPB*, kudu::rpc::ErrorStatusPB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:521:5 (libtserver.so+0x1cd4c4)
    #32 kudu::tserver::Heartbeater::Thread::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:645:21 (libtserver.so+0x1ce790)
    #33 kudu::tserver::Heartbeater::Thread::Start()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:728:48 (libtserver.so+0x1d03c1)
    #34 decltype(std::__1::forward<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(fp)()) std::__1::__invoke<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x1d0379)
    #35 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x1d0309)
    #36 std::__1::__function::__alloc_func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x1d02d1)
    #37 std::__1::__function::__func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x1cf5cd)
    #38 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #39 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #40 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Previous atomic write of size 8 at 0x7b44000b4578 by thread T144 (mutexes: write M299906313465758112):
    #0 __tsan_atomic64_compare_exchange_val /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:850 (kudu+0x2d19c8)
    #1 bool std::__1::__cxx_atomic_compare_exchange_weak<long>(std::__1::__cxx_atomic_base_impl<long>*, long*, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1031:12 (liblog.so+0x8c014)
    #2 std::__1::__atomic_base<long, false>::compare_exchange_weak(long&, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1622:17 (liblog.so+0x8be39)
    #3 void kudu::AtomicStoreMax<long>(std::__1::atomic<long>&, long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/atomic-utils.h:33:13 (liblog.so+0x7ec80)
    #4 kudu::HighWaterMark::UpdateMax(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:80:5 (libkudu_util.so+0x380794)
    #5 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:5 (libkudu_util.so+0x37b5fd)
    #6 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x378c61)
    #7 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x451230)
    #8 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:608:16 (libtablet.so+0x44a1c2)
    #9 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:579:5 (libtablet.so+0x449953)
    #10 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:65 (libtablet.so+0x44db61)
    #11 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x44db19)
    #12 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x44daa9)
    #13 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x44da71)
    #14 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x44cc9d)
    #15 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #16 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #17 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x4403a5)
    #18 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x442711)
    #19 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x4426c9)
    #20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x442659)
    #21 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x442621)
    #22 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x44191d)
    #23 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #24 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #25 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

I20261120 03:00:52.955022  1180 tablet_bootstrap.cc:700] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440: Time spent opening tablet: real 0.153s	user 0.004s	sys 0.000s
I20261120 03:00:52.890875  1400 raft_consensus.cc:556] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:52.959445  1400 leader_election.cc:290] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:52.904310  1185 tablet_replica.cc:333] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:52.934953  1289 leader_election.cc:290] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:52.960928  1185 raft_consensus.cc:2440] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:52.961850  1185 raft_consensus.cc:2469] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
  Mutex M299906313465758112 is already destroyed.

  Thread T127 'heartbeat-1124' (tid=1124, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::tserver::Heartbeater::Thread::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:727:10 (libtserver.so+0x1cae61)
    #4 kudu::tserver::Heartbeater::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:251:41 (libtserver.so+0x1cac3d)
    #5 kudu::tserver::TabletServer::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:167:3 (libtserver.so+0x259ccc)
    #6 kudu::tserver::RunTabletServer() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server_runner.cc:86:3 (libtserver.so+0x266e93)
    #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:128:10 (libkudu_cli_actions.so+0x3b6429)
    #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_cli_actions.so+0x28c051)
    #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (libkudu_cli_actions.so+0x28bfbc)
    #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_cli_actions.so+0x28bf2c)
    #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_cli_actions.so+0x28ab95)
    #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x14082f)
    #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132da1)
    #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:270:10 (libkudu_tools_util.so+0x130a52)
    #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:114:15 (kudu+0x324910)
    #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:186:16 (kudu+0x325247)
    #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:249:10 (kudu+0x325671)

  Thread T144 'apply [worker]-' (tid=1286, finished) created by thread T82 at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x43f5cb)
    #4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x43d8cb)
    #5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x43fb32)
    #6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:23 (libtablet.so+0x448cf0)
    #7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:476:5 (libtablet.so+0x449289)
    #8 kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:860:15 (libtablet.so+0x41362a)
    #9 decltype(std::__1::forward<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x4135b8)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x413538)
    #11 std::__1::__function::__alloc_func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x4134f3)
    #12 std::__1::__function::__func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41270c)
    #13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x40fe06)
    #14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x40a5b8)
    #15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3598:3 (libconsensus.so+0x1fd9c0)
    #16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:200:12 (libconsensus.so+0x1d2e9d)
    #17 kudu::consensus::RaftConsensus::UpdateReplica(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::consensus::RaftConsensus::FlushContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1603:5 (libconsensus.so+0x1f0713)
    #18 kudu::consensus::RaftConsensus::Update(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1162:9 (libconsensus.so+0x1efd04)
    #19 kudu::tserver::ConsensusServiceImpl::UpdateConsensus(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1792:25 (libtserver.so+0x26e199)
    #20 kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/consensus/consensus.service.cc:321:13 (libconsensus_proto.so+0xd8d34)
    #21 decltype(std::__1::forward<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus_proto.so+0xd8cc2)
    #22 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus_proto.so+0xd8bf1)
    #23 std::__1::__function::__alloc_func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus_proto.so+0xd8b6c)
    #24 std::__1::__function::__func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus_proto.so+0xd7e22)
    #25 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x21b0bc)
    #26 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x21a4f6)
    #27 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x219e9f)
    #28 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15 (libkrpc.so+0x21d16f)
    #29 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5 (libkrpc.so+0x21e4c1)
    #30 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x21e479)
    #31 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x21e409)
    #32 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x21e3d1)
    #33 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x21d6cd)
    #34 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #35 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #36 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
==================
I20261120 03:00:52.965420  1045 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 68179488b9bc4e03b65a9f1d434c95b5. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.965924  1185 ts_tablet_manager.cc:1962] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:52.969583  1046 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1885397e42774a89bb347bc776284bc0. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.946372   710 leader_election.cc:304] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:52.970458  1289 raft_consensus.cc:3024] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:52.970881  1289 raft_consensus.cc:744] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:52.971704  1047 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet daae644853734cd5818a46ecaa5ce7df. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:52.971629  1289 consensus_queue.cc:238] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
W20261120 03:00:52.978164   709 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 88629.2us
W20261120 03:00:52.978755   709 leader_election.cc:343] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:52.979476   709 leader_election.cc:343] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Not found: Tablet not found: a1f6197c60dd4dfb8aeb5c38373aab21
W20261120 03:00:52.980170   709 leader_election.cc:343] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:52.981482  1180 tablet_bootstrap.cc:654] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:52.976017   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5a3fe1499d0b4fab9f9949c8ded0a8dc" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:52.983008   923 raft_consensus.cc:3296] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:52.986354  1392 tablet_replica.cc:333] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:52.987171  1392 raft_consensus.cc:2440] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:52.987653  1392 raft_consensus.cc:2469] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:52.989554  1392 ts_tablet_manager.cc:1962] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:52.989976   923 raft_consensus.cc:2062] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:52.949434  1324 tablet_bootstrap.cc:654] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:52.994138  1059 thread.cc:642] tablet-delete [worker] (thread pool) Time spent creating pthread: real 0.739s	user 0.034s	sys 0.038s
W20261120 03:00:52.994421  1059 thread.cc:609] tablet-delete [worker] (thread pool) Time spent starting thread: real 0.739s	user 0.034s	sys 0.038s
I20261120 03:00:52.996904   710 leader_election.cc:304] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:52.997780  1400 raft_consensus.cc:3024] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:52.998219  1400 raft_consensus.cc:744] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:52.999001  1400 consensus_queue.cc:238] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:53.014708   900 tablet_service.cc:1495] Processing CreateTablet for tablet 0db7db25d779459ebde532f938493975 (DEFAULT_TABLE table=default.Table_9498dfca4c8544c0a19cd826d27d441e [id=f3b5f7a4e5494e82a472d0025df96e0c]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.015810   900 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0db7db25d779459ebde532f938493975. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.028332   751 tablet_service.cc:1495] Processing CreateTablet for tablet 0db7db25d779459ebde532f938493975 (DEFAULT_TABLE table=default.Table_9498dfca4c8544c0a19cd826d27d441e [id=f3b5f7a4e5494e82a472d0025df96e0c]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.029453   751 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0db7db25d779459ebde532f938493975. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.041034  1405 consensus_queue.cc:1049] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20261120 03:00:53.045764   493 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 200b234da897419a82bf5844620adf24
I20261120 03:00:53.049887   903 tablet_service.cc:1542] Processing DeleteTablet for tablet 200b234da897419a82bf5844620adf24 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:52 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:53.052086   753 tablet_service.cc:1542] Processing DeleteTablet for tablet 200b234da897419a82bf5844620adf24 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:52 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:53.053418  1059 tablet_service.cc:1495] Processing CreateTablet for tablet 0db7db25d779459ebde532f938493975 (DEFAULT_TABLE table=default.Table_9498dfca4c8544c0a19cd826d27d441e [id=f3b5f7a4e5494e82a472d0025df96e0c]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.054296  1405 consensus_queue.cc:1049] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20261120 03:00:53.059657   709 leader_election.cc:343] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Not found: Tablet not found: 4abb67f3819a45b9b05dde04842e2c9d
W20261120 03:00:53.060523   709 consensus_peers.cc:597] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a1f6197c60dd4dfb8aeb5c38373aab21. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:53.061353  1059 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0db7db25d779459ebde532f938493975. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.066769  1048 tablet_service.cc:1495] Processing CreateTablet for tablet d736d65c3e0b419084b4934dad83f16a (DEFAULT_TABLE table=default.Table_04ddab7f706b43e7ba12fa247b3ca933 [id=9f571a69327a498e89f323c9ac0eee34]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.069156   499 catalog_manager.cc:5856] T a1f6197c60dd4dfb8aeb5c38373aab21 P 5beeefe8bef54caeac463d05d8c30fa6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65). New cstate: current_term: 1 leader_uuid: "5beeefe8bef54caeac463d05d8c30fa6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: HEALTHY } } }
I20261120 03:00:53.070858  1324 tablet_bootstrap.cc:492] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:53.071238  1324 ts_tablet_manager.cc:1449] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.242s	user 0.009s	sys 0.000s
I20261120 03:00:53.071632   499 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 5fee8dff291c43e9be0fb10b308718ce (table default.Table_a1f866996679431e9e1c4c5f6f8cdd12 [id=a275e215155b4824a1494f0954577fe8]). Expected version 1 got 0
I20261120 03:00:53.073536  1324 raft_consensus.cc:366] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.073875   497 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 00dd2ffad1124244b171b2657d38a708
I20261120 03:00:53.077363  1050 tablet_service.cc:1542] Processing DeleteTablet for tablet 200b234da897419a82bf5844620adf24 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:52 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:53.078526  1048 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d736d65c3e0b419084b4934dad83f16a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.095804  1072 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5a3fe1499d0b4fab9f9949c8ded0a8dc" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:53.097939  1050 tablet_service.cc:1542] Processing DeleteTablet for tablet 00dd2ffad1124244b171b2657d38a708 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:52 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:53.101994  1072 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "200b234da897419a82bf5844620adf24" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:53.106346  1295 ts_tablet_manager.cc:1975] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20261120 03:00:53.106768  1295 log.cc:1199] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/5f5c8711a6eb43c3aff1c27ed5115d6a
I20261120 03:00:53.107424  1295 ts_tablet_manager.cc:1996] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:53.115481  1072 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "200b234da897419a82bf5844620adf24" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:53.118707  1072 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5a3fe1499d0b4fab9f9949c8ded0a8dc" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:53.119676   499 catalog_manager.cc:5856] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65). New cstate: current_term: 1 leader_uuid: "5beeefe8bef54caeac463d05d8c30fa6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: HEALTHY } } }
I20261120 03:00:53.127796  1180 tablet_bootstrap.cc:492] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:53.130733  1180 ts_tablet_manager.cc:1449] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.333s	user 0.012s	sys 0.002s
W20261120 03:00:53.137881  1427 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:00:53.141104  1180 raft_consensus.cc:366] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:53.083969   753 tablet_service.cc:1542] Processing DeleteTablet for tablet 00dd2ffad1124244b171b2657d38a708 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:52 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:53.099809   750 tablet_service.cc:1495] Processing CreateTablet for tablet d736d65c3e0b419084b4934dad83f16a (DEFAULT_TABLE table=default.Table_04ddab7f706b43e7ba12fa247b3ca933 [id=9f571a69327a498e89f323c9ac0eee34]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.100590   755 tablet_service.cc:1495] Processing CreateTablet for tablet 35f0ff39b36148e8abfdea3435a3b8b4 (DEFAULT_TABLE table=default.Table_54b560cd2410426782bbd02c54839db2 [id=08745a3ff89a4b39ab3a8886a18ebc9b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.137784   495 master_service.cc:965] ReplaceTablet: received request to replace tablet 1885397e42774a89bb347bc776284bc0 from {username='slave'} at 127.0.0.1:45640
I20261120 03:00:53.143081   750 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d736d65c3e0b419084b4934dad83f16a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.145805  1050 tablet_service.cc:1495] Processing CreateTablet for tablet 35f0ff39b36148e8abfdea3435a3b8b4 (DEFAULT_TABLE table=default.Table_54b560cd2410426782bbd02c54839db2 [id=08745a3ff89a4b39ab3a8886a18ebc9b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.146458   755 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 35f0ff39b36148e8abfdea3435a3b8b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.146863  1050 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 35f0ff39b36148e8abfdea3435a3b8b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.166582  1324 raft_consensus.cc:392] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:53.166922  1324 raft_consensus.cc:787] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:53.167567  1324 consensus_queue.cc:261] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.169983  1324 ts_tablet_manager.cc:1480] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.099s	user 0.007s	sys 0.000s
I20261120 03:00:53.170847  1324 tablet_bootstrap.cc:492] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:53.170887  1048 tablet_service.cc:1495] Processing CreateTablet for tablet b6f0ff2246cd4772af47812b699e93e8 (DEFAULT_TABLE table=default.Table_eccdaea2421f42fd95a4253870c13623 [id=606c1f1f5a8541e5be8cff24b2a4b668]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.172137  1048 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b6f0ff2246cd4772af47812b699e93e8. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:00:53.174500   709 consensus_peers.cc:597] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
W20261120 03:00:53.175031   709 leader_election.cc:343] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:53.175791   709 consensus_peers.cc:597] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
W20261120 03:00:53.176239   709 leader_election.cc:343] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:53.176932   709 leader_election.cc:343] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:53.177620   709 leader_election.cc:343] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:53.181845   753 tablet_service.cc:1495] Processing CreateTablet for tablet b6f0ff2246cd4772af47812b699e93e8 (DEFAULT_TABLE table=default.Table_eccdaea2421f42fd95a4253870c13623 [id=606c1f1f5a8541e5be8cff24b2a4b668]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.182945   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b6f0ff2246cd4772af47812b699e93e8. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.194094  1185 ts_tablet_manager.cc:1975] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:53.203827   500 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_975b3ffda1cd474c8d82a00487c79adc" } new_table_name: "default.Table_a925e1ca8a3d4ee18ad6481a8e3c1a73" modify_external_catalogs: true
I20261120 03:00:53.204223   496 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_0c88eb71972c46cd918b4abf3ee01e11"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:53.206095   496 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_0c88eb71972c46cd918b4abf3ee01e11 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:53.206250  1185 log.cc:1199] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/fa57d0c282964e37b9133ea57f458a51
I20261120 03:00:53.206928  1185 ts_tablet_manager.cc:1996] T fa57d0c282964e37b9133ea57f458a51 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
W20261120 03:00:53.196854   709 consensus_peers.cc:597] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:53.209419  1392 ts_tablet_manager.cc:1975] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
W20261120 03:00:53.209720   709 consensus_peers.cc:597] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:53.209775  1392 log.cc:1199] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/a72d5d7896de4020a1ac4d2f8c551e15
I20261120 03:00:53.209641   494 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_eccdaea2421f42fd95a4253870c13623" } new_table_name: "default.Table_28990e82c07a47a29f4dbc3da17c42bb" modify_external_catalogs: true
I20261120 03:00:53.210516   497 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_fbcff6b6a1aa454aaf2cd73f3b7381b0"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:53.212150   497 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_fbcff6b6a1aa454aaf2cd73f3b7381b0 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:53.212724  1324 tablet_bootstrap.cc:654] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
==================
WARNING: ThreadSanitizer: data race (pid=977)
  Write of size 8 at 0x7b44000b4e30 by thread T127:
    #0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (kudu+0x321819)
I20261120 03:00:53.216331  1392 ts_tablet_manager.cc:1996] T a72d5d7896de4020a1ac4d2f8c551e15 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:53.219919   501 master_service.cc:965] ReplaceTablet: received request to replace tablet 020be7f7cda0467aab56f657cbe8d0ad from {username='slave'} at 127.0.0.1:45640
    #1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (kudu+0x3287c9)
    #2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (kudu+0x328769)
    #3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x2aebd2)
    #4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x2aeb69)
    #5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x381bcd)
    #6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x381500)
    #7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
    #8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libkudu_cli_actions.so+0x24d1d5)
    #9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x450864)
    #11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x403c45)
    #12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x403e59)
    #13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libkudu_cli_actions.so+0x2fbea7)
    #14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libkudu_cli_actions.so+0x2fbe69)
    #15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libkudu_cli_actions.so+0x2fbe29)
I20261120 03:00:53.223577  1185 tablet_replica.cc:333] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
    #16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libkudu_cli_actions.so+0x2fa1fa)
    #17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3e1f09)
    #18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3e1ee0)
I20261120 03:00:53.224277  1185 raft_consensus.cc:2440] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Raft consensus shutting down.
    #19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3e1ea0)
I20261120 03:00:53.224895  1185 pending_rounds.cc:70] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: Trying to abort 1 pending ops.
    #20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b4aa6)
    #21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b49c4)
I20261120 03:00:53.224547  1392 tablet_replica.cc:333] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:53.227474   493 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_a97b43669b2d4944ad7c61d64e5a2cbd"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:53.229270   493 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_a97b43669b2d4944ad7c61d64e5a2cbd in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
    #22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b477b)
I20261120 03:00:53.230686  1185 pending_rounds.cc:77] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7352908402572042240 op_type: NO_OP noop_request { }
    #23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x28c761)
    #24 kudu::tserver::TSTabletManager::PopulateIncrementalTabletReport(kudu::master::TabletReportPB*, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, bool) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1893:1 (libtserver.so+0x2c1b50)
I20261120 03:00:53.234473  1185 raft_consensus.cc:3115] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
    #25 kudu::tserver::Heartbeater::Thread::GenerateIncrementalTabletReport(kudu::master::TabletReportPB*, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:790:30 (libtserver.so+0x1cb58a)
    #26 kudu::tserver::Heartbeater::Thread::DoHeartbeat(kudu::master::MasterErrorPB*, kudu::rpc::ErrorStatusPB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:521:5 (libtserver.so+0x1cd4c4)
I20261120 03:00:53.234943  1185 raft_consensus.cc:2469] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
    #27 kudu::tserver::Heartbeater::Thread::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:645:21 (libtserver.so+0x1ce790)
    #28 kudu::tserver::Heartbeater::Thread::Start()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:728:48 (libtserver.so+0x1d03c1)
    #29 decltype(std::__1::forward<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(fp)()) std::__1::__invoke<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x1d0379)
    #30 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x1d0309)
    #31 std::__1::__function::__alloc_func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x1d02d1)
    #32 std::__1::__function::__func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x1cf5cd)
    #33 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #34 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #35 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Previous atomic write of size 8 at 0x7b44000b4e30 by thread T144 (mutexes: write M299906313465758112):
    #0 __tsan_atomic64_fetch_add /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:620 (kudu+0x2ca8f5)
    #1 long std::__1::__cxx_atomic_fetch_add<long>(std::__1::__cxx_atomic_base_impl<long>*, long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1042:12 (libkudu_util.so+0x2c2cd3)
    #2 std::__1::__atomic_base<long, true>::fetch_add(long, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1706:17 (libkudu_util.so+0x2c2c09)
I20261120 03:00:53.237917  1185 ts_tablet_manager.cc:1962] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
    #3 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:30 (libkudu_util.so+0x37b5f2)
    #4 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x378c61)
    #5 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x451230)
    #6 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:608:16 (libtablet.so+0x44a1c2)
    #7 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:579:5 (libtablet.so+0x449953)
    #8 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:65 (libtablet.so+0x44db61)
    #9 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x44db19)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x44daa9)
    #11 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x44da71)
    #12 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x44cc9d)
    #13 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #14 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #15 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x4403a5)
    #16 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x442711)
    #17 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x4426c9)
    #18 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x442659)
    #19 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x442621)
    #20 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x44191d)
    #21 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #22 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #23 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Mutex M299906313465758112 is already destroyed.

  Thread T127 'heartbeat-1124' (tid=1124, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::tserver::Heartbeater::Thread::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:727:10 (libtserver.so+0x1cae61)
I20261120 03:00:53.244670   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet fa57d0c282964e37b9133ea57f458a51 (table default.Table_dd2f7617ff0e401b9b862d5991110dd5 [id=3955f41b6d4d41bf8ba2c81e4ac7f4a8]) successfully deleted
    #4 kudu::tserver::Heartbeater::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:251:41 (libtserver.so+0x1cac3d)
I20261120 03:00:53.244541  1392 raft_consensus.cc:2440] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
    #5 kudu::tserver::TabletServer::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:167:3 (libtserver.so+0x259ccc)
    #6 kudu::tserver::RunTabletServer() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server_runner.cc:86:3 (libtserver.so+0x266e93)
    #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:128:10 (libkudu_cli_actions.so+0x3b6429)
    #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_cli_actions.so+0x28c051)
I20261120 03:00:53.245923  1392 raft_consensus.cc:2469] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:53.246049   499 catalog_manager.cc:5856] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 5beeefe8bef54caeac463d05d8c30fa6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65). New cstate: current_term: 1 leader_uuid: "5beeefe8bef54caeac463d05d8c30fa6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: UNKNOWN } } }
    #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (libkudu_cli_actions.so+0x28bfbc)
    #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_cli_actions.so+0x28bf2c)
    #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_cli_actions.so+0x28ab95)
    #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x14082f)
    #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132da1)
    #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:270:10 (libkudu_tools_util.so+0x130a52)
    #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:114:15 (kudu+0x324910)
    #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:186:16 (kudu+0x325247)
    #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:249:10 (kudu+0x325671)

  Thread T144 'apply [worker]-' (tid=1286, finished) created by thread T82 at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x43f5cb)
    #4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x43d8cb)
    #5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x43fb32)
    #6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:23 (libtablet.so+0x448cf0)
    #7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:476:5 (libtablet.so+0x449289)
    #8 kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:860:15 (libtablet.so+0x41362a)
I20261120 03:00:53.263901  1324 tablet_bootstrap.cc:492] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:53.264331  1324 ts_tablet_manager.cc:1449] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.094s	user 0.009s	sys 0.000s
I20261120 03:00:53.266639  1324 raft_consensus.cc:366] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.267928  1324 raft_consensus.cc:392] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:53.268535  1324 raft_consensus.cc:787] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:53.248934   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet a72d5d7896de4020a1ac4d2f8c551e15 (table default.Table_f601bffe6ddb49b3a3c589ac7b25235c [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]) successfully deleted
I20261120 03:00:53.271673   498 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_5c5911c8746d4b26843ec77c8141d3b6" } modify_external_catalogs: true
I20261120 03:00:53.272184   498 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_5c5911c8746d4b26843ec77c8141d3b6" } modify_external_catalogs: true
    #9 decltype(std::__1::forward<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x4135b8)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x413538)
I20261120 03:00:53.263469  1392 ts_tablet_manager.cc:1962] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
    #11 std::__1::__function::__alloc_func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x4134f3)
    #12 std::__1::__function::__func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41270c)
    #13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x40fe06)
W20261120 03:00:53.269332   486 catalog_manager.cc:5061] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet df61317717d3401eb74cd93aa336c4dc because tablet deleting was already in progress. No further retry: Already present: State transition of tablet df61317717d3401eb74cd93aa336c4dc already in progress: creating tablet
    #14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x40a5b8)
    #15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3598:3 (libconsensus.so+0x1fd9c0)
    #16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:200:12 (libconsensus.so+0x1d2e9d)
    #17 kudu::consensus::RaftConsensus::UpdateReplica(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::consensus::RaftConsensus::FlushContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1603:5 (libconsensus.so+0x1f0713)
    #18 kudu::consensus::RaftConsensus::Update(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1162:9 (libconsensus.so+0x1efd04)
    #19 kudu::tserver::ConsensusServiceImpl::UpdateConsensus(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1792:25 (libtserver.so+0x26e199)
    #20 kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/consensus/consensus.service.cc:321:13 (libconsensus_proto.so+0xd8d34)
    #21 decltype(std::__1::forward<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus_proto.so+0xd8cc2)
    #22 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus_proto.so+0xd8bf1)
    #23 std::__1::__function::__alloc_func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus_proto.so+0xd8b6c)
    #24 std::__1::__function::__func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus_proto.so+0xd7e22)
    #25 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x21b0bc)
    #26 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x21a4f6)
    #27 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x219e9f)
    #28 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15 (libkrpc.so+0x21d16f)
    #29 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5 (libkrpc.so+0x21e4c1)
    #30 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x21e479)
    #31 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x21e409)
    #32 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x21e3d1)
    #33 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x21d6cd)
    #34 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #35 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #36 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
==================
I20261120 03:00:53.284894  1324 consensus_queue.cc:261] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.282477   502 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_3c79e29085ad4875874397eaefcab250" } modify_external_catalogs: true
I20261120 03:00:53.287938   502 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_3c79e29085ad4875874397eaefcab250" } modify_external_catalogs: true
I20261120 03:00:53.291714  1289 raft_consensus.cc:534] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:53.292158  1289 raft_consensus.cc:556] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.294636  1289 leader_election.cc:290] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:53.296684  1072 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d27099bc667b4a1d880af7cec2715bb8" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:53.298204   709 leader_election.cc:343] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:53.304478  1324 ts_tablet_manager.cc:1480] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.040s	user 0.006s	sys 0.000s
I20261120 03:00:53.333132  1392 ts_tablet_manager.cc:1975] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:53.335346  1392 log.cc:1199] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/3f65ba863447413cb122e728df3632fd
I20261120 03:00:53.335996  1392 ts_tablet_manager.cc:1996] T 3f65ba863447413cb122e728df3632fd P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:53.343909   495 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 1885397e42774a89bb347bc776284bc0
I20261120 03:00:53.346313   495 catalog_manager.cc:6952] ReplaceTablet: tablet 1885397e42774a89bb347bc776284bc0 deleted and replaced by tablet b549a06d15db449b851fe9549a963416
I20261120 03:00:53.348464   495 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_db0de8e58235444da0d82057b21e8b0f"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:53.361110  1324 tablet_bootstrap.cc:492] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:53.376156  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 45) took 1069 ms. Trace:
I20261120 03:00:53.376619  1392 rpcz_store.cc:276] 1120 03:00:52.307547 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:52.307803 (+   256us) service_pool.cc:224] Handling call
1120 03:00:53.223803 (+916000us) ts_tablet_manager.cc:1197] Deleting tablet 3f65ba863447413cb122e728df3632fd
1120 03:00:53.223978 (+   175us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:53.300418 (+ 76440us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:53.333003 (+ 32585us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:53.376087 (+ 43084us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":8058,"tablet-delete.queue_time_us":911474}
W20261120 03:00:53.379258   495 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_db0de8e58235444da0d82057b21e8b0f in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:53.313866   899 tablet_service.cc:1495] Processing CreateTablet for tablet b6f0ff2246cd4772af47812b699e93e8 (DEFAULT_TABLE table=default.Table_eccdaea2421f42fd95a4253870c13623 [id=606c1f1f5a8541e5be8cff24b2a4b668]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.390100   899 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b6f0ff2246cd4772af47812b699e93e8. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.315546   901 tablet_service.cc:1495] Processing CreateTablet for tablet d736d65c3e0b419084b4934dad83f16a (DEFAULT_TABLE table=default.Table_04ddab7f706b43e7ba12fa247b3ca933 [id=9f571a69327a498e89f323c9ac0eee34]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.395385   901 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet d736d65c3e0b419084b4934dad83f16a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.316783   902 tablet_service.cc:1495] Processing CreateTablet for tablet 35f0ff39b36148e8abfdea3435a3b8b4 (DEFAULT_TABLE table=default.Table_54b560cd2410426782bbd02c54839db2 [id=08745a3ff89a4b39ab3a8886a18ebc9b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.399499   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 35f0ff39b36148e8abfdea3435a3b8b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.317510   903 tablet_service.cc:1542] Processing DeleteTablet for tablet 00dd2ffad1124244b171b2657d38a708 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:52 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:53.336315   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d27099bc667b4a1d880af7cec2715bb8" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:53.401641   924 raft_consensus.cc:2062] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:53.186515  1295 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 57) took 1359 ms. Trace:
I20261120 03:00:53.403615  1324 tablet_bootstrap.cc:654] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:53.402622  1295 rpcz_store.cc:276] 1120 03:00:51.826809 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:51.826947 (+   138us) service_pool.cc:224] Handling call
1120 03:00:52.772271 (+945324us) ts_tablet_manager.cc:1197] Deleting tablet 5f5c8711a6eb43c3aff1c27ed5115d6a
1120 03:00:52.772306 (+    35us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:53.009509 (+237203us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:53.106204 (+ 96695us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:53.186450 (+ 80246us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":900969}
I20261120 03:00:53.404656  1295 tablet_replica.cc:333] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:53.356356   904 tablet_service.cc:1542] Processing DeleteTablet for tablet 1885397e42774a89bb347bc776284bc0 with delete_type TABLET_DATA_DELETED (replaced by tablet b549a06d15db449b851fe9549a963416) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:53.406620  1295 raft_consensus.cc:2440] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:53.407320  1295 raft_consensus.cc:2469] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
==================
WARNING: ThreadSanitizer: data race (pid=977)
  Write of size 8 at 0x7b44000b4e38 by thread T127:
    #0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (kudu+0x321819)
    #1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (kudu+0x3287c9)
    #2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (kudu+0x328769)
    #3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x2aebd2)
    #4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x2aeb69)
    #5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x381bcd)
    #6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x381500)
    #7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
    #8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libkudu_cli_actions.so+0x24d1d5)
    #9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x450864)
    #11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x403c45)
    #12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x403e59)
    #13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libkudu_cli_actions.so+0x2fbea7)
    #14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libkudu_cli_actions.so+0x2fbe69)
    #15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libkudu_cli_actions.so+0x2fbe29)
    #16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libkudu_cli_actions.so+0x2fa1fa)
    #17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3e1f09)
    #18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3e1ee0)
    #19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3e1ea0)
    #20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b4aa6)
    #21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b49c4)
    #22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b477b)
    #23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x28c761)
    #24 kudu::tserver::TSTabletManager::PopulateIncrementalTabletReport(kudu::master::TabletReportPB*, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, bool) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1893:1 (libtserver.so+0x2c1b50)
    #25 kudu::tserver::Heartbeater::Thread::GenerateIncrementalTabletReport(kudu::master::TabletReportPB*, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:790:30 (libtserver.so+0x1cb58a)
    #26 kudu::tserver::Heartbeater::Thread::DoHeartbeat(kudu::master::MasterErrorPB*, kudu::rpc::ErrorStatusPB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:521:5 (libtserver.so+0x1cd4c4)
    #27 kudu::tserver::Heartbeater::Thread::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:645:21 (libtserver.so+0x1ce790)
    #28 kudu::tserver::Heartbeater::Thread::Start()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:728:48 (libtserver.so+0x1d03c1)
    #29 decltype(std::__1::forward<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(fp)()) std::__1::__invoke<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x1d0379)
    #30 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x1d0309)
    #31 std::__1::__function::__alloc_func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x1d02d1)
    #32 std::__1::__function::__func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x1cf5cd)
    #33 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #34 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #35 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Previous atomic write of size 8 at 0x7b44000b4e38 by thread T144 (mutexes: write M299906313465758112):
    #0 __tsan_atomic64_compare_exchange_val /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:850 (kudu+0x2d19c8)
    #1 bool std::__1::__cxx_atomic_compare_exchange_weak<long>(std::__1::__cxx_atomic_base_impl<long>*, long*, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1031:12 (liblog.so+0x8c014)
    #2 std::__1::__atomic_base<long, false>::compare_exchange_weak(long&, long, std::__1::memory_order, std::__1::memory_order) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/atomic:1622:17 (liblog.so+0x8be39)
    #3 void kudu::AtomicStoreMax<long>(std::__1::atomic<long>&, long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/atomic-utils.h:33:13 (liblog.so+0x7ec80)
    #4 kudu::HighWaterMark::UpdateMax(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:80:5 (libkudu_util.so+0x380794)
    #5 kudu::HighWaterMark::IncrementBy(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/high_water_mark.h:70:5 (libkudu_util.so+0x37b5fd)
    #6 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:285:27 (libkudu_util.so+0x378c61)
    #7 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x451230)
    #8 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:608:16 (libtablet.so+0x44a1c2)
    #9 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:579:5 (libtablet.so+0x449953)
    #10 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:65 (libtablet.so+0x44db61)
    #11 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x44db19)
    #12 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x44daa9)
    #13 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x44da71)
    #14 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x44cc9d)
    #15 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
I20261120 03:00:53.411835  1295 ts_tablet_manager.cc:1962] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
    #16 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #17 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x4403a5)
    #18 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x442711)
    #19 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x4426c9)
    #20 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x442659)
    #21 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x442621)
    #22 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x44191d)
    #23 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #24 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #25 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Mutex M299906313465758112 is already destroyed.

  Thread T127 'heartbeat-1124' (tid=1124, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
I20261120 03:00:53.418100   710 leader_election.cc:304] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::tserver::Heartbeater::Thread::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:727:10 (libtserver.so+0x1cae61)
    #4 kudu::tserver::Heartbeater::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:251:41 (libtserver.so+0x1cac3d)
I20261120 03:00:53.419337  1289 raft_consensus.cc:3024] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
    #5 kudu::tserver::TabletServer::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:167:3 (libtserver.so+0x259ccc)
    #6 kudu::tserver::RunTabletServer() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server_runner.cc:86:3 (libtserver.so+0x266e93)
    #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:128:10 (libkudu_cli_actions.so+0x3b6429)
I20261120 03:00:53.419744  1289 raft_consensus.cc:534] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
    #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_cli_actions.so+0x28c051)
I20261120 03:00:53.420060  1289 raft_consensus.cc:3296] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
    #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (libkudu_cli_actions.so+0x28bfbc)
    #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_cli_actions.so+0x28bf2c)
    #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_cli_actions.so+0x28ab95)
    #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x14082f)
    #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132da1)
    #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:270:10 (libkudu_tools_util.so+0x130a52)
    #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:114:15 (kudu+0x324910)
    #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:186:16 (kudu+0x325247)
    #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:249:10 (kudu+0x325671)

  Thread T144 'apply [worker]-' (tid=1286, finished) created by thread T82 at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x43f5cb)
    #4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x43d8cb)
    #5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x43fb32)
    #6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:23 (libtablet.so+0x448cf0)
    #7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:476:5 (libtablet.so+0x449289)
    #8 kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:860:15 (libtablet.so+0x41362a)
    #9 decltype(std::__1::forward<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x4135b8)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x413538)
    #11 std::__1::__function::__alloc_func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x4134f3)
    #12 std::__1::__function::__func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41270c)
    #13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x40fe06)
    #14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x40a5b8)
    #15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3598:3 (libconsensus.so+0x1fd9c0)
    #16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:200:12 (libconsensus.so+0x1d2e9d)
    #17 kudu::consensus::RaftConsensus::UpdateReplica(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::consensus::RaftConsensus::FlushContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1603:5 (libconsensus.so+0x1f0713)
    #18 kudu::consensus::RaftConsensus::Update(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1162:9 (libconsensus.so+0x1efd04)
    #19 kudu::tserver::ConsensusServiceImpl::UpdateConsensus(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1792:25 (libtserver.so+0x26e199)
    #20 kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/consensus/consensus.service.cc:321:13 (libconsensus_proto.so+0xd8d34)
    #21 decltype(std::__1::forward<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus_proto.so+0xd8cc2)
    #22 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus_proto.so+0xd8bf1)
    #23 std::__1::__function::__alloc_func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus_proto.so+0xd8b6c)
    #24 std::__1::__function::__func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus_proto.so+0xd7e22)
    #25 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x21b0bc)
    #26 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x21a4f6)
    #27 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x219e9f)
    #28 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15 (libkrpc.so+0x21d16f)
    #29 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5 (libkrpc.so+0x21e4c1)
    #30 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x21e479)
    #31 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x21e409)
    #32 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x21e3d1)
    #33 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x21d6cd)
    #34 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #35 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #36 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
==================
I20261120 03:00:53.420841  1289 raft_consensus.cc:556] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
W20261120 03:00:53.433394  1181 thread.cc:642] log-alloc [worker] (thread pool) Time spent creating pthread: real 0.576s	user 0.037s	sys 0.042s
W20261120 03:00:53.434552  1181 thread.cc:609] log-alloc [worker] (thread pool) Time spent starting thread: real 0.577s	user 0.037s	sys 0.042s
W20261120 03:00:53.434140  1382 tablet_replica.cc:1411] Time spent flushing the Log queue took a long time: real 0.463s	user 0.028s	sys 0.049s
I20261120 03:00:53.443475  1181 tablet_bootstrap.cc:492] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:53.454288  1181 ts_tablet_manager.cc:1449] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.701s	user 0.046s	sys 0.042s
I20261120 03:00:53.454630  1180 raft_consensus.cc:392] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:53.455014  1180 raft_consensus.cc:787] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:53.458786  1392 tablet_replica.cc:333] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:53.459638  1392 raft_consensus.cc:2440] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:53.460312  1392 raft_consensus.cc:2469] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:53.460590  1181 raft_consensus.cc:366] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.464879  1185 ts_tablet_manager.cc:1975] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:53.465266  1185 log.cc:1199] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/200b234da897419a82bf5844620adf24
I20261120 03:00:53.465946  1185 ts_tablet_manager.cc:1996] T 200b234da897419a82bf5844620adf24 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:53.467829  1289 leader_election.cc:290] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:53.474059  1185 tablet_replica.cc:333] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:53.476953   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 3f65ba863447413cb122e728df3632fd (table default.Table_4755bb4a11ae4e33bff2a6cc7384921a [id=676ae8fc1a8c488d820a11b930b0ed5f]) successfully deleted
I20261120 03:00:53.470124  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d27099bc667b4a1d880af7cec2715bb8" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:53.477996   753 tablet_service.cc:1542] Processing DeleteTablet for tablet 1885397e42774a89bb347bc776284bc0 with delete_type TABLET_DATA_DELETED (replaced by tablet b549a06d15db449b851fe9549a963416) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:53.475538  1048 tablet_service.cc:1495] Processing CreateTablet for tablet 168cb98bb1b242a5a0377bf292204d96 (DEFAULT_TABLE table=default.Table_3adc64475e544d4e9c19ed6354bdc9be [id=cdf4cd19161e41daa9626fdde8c99699]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.479414  1048 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 168cb98bb1b242a5a0377bf292204d96. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.486008  1050 tablet_service.cc:1542] Processing DeleteTablet for tablet 1885397e42774a89bb347bc776284bc0 with delete_type TABLET_DATA_DELETED (replaced by tablet b549a06d15db449b851fe9549a963416) from {username='slave'} at 127.0.0.1:50744
W20261120 03:00:53.486974   709 leader_election.cc:343] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:53.489778  1180 consensus_queue.cc:261] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:53.493633  1180 ts_tablet_manager.cc:1480] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.355s	user 0.005s	sys 0.000s
I20261120 03:00:53.494211  1181 raft_consensus.cc:392] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:53.495276  1180 tablet_bootstrap.cc:492] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:53.500782  1280 raft_consensus.cc:534] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:53.501252  1280 raft_consensus.cc:556] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.501961   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 200b234da897419a82bf5844620adf24 (table default.Table_5303db2882934be1bfc2251ce51f2c8d [id=ae52d3d892e24bc38a8ff7d6db755c2d]) successfully deleted
I20261120 03:00:53.494910  1181 raft_consensus.cc:787] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:53.503212  1280 leader_election.cc:290] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:53.503392  1181 consensus_queue.cc:261] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.504792   753 tablet_service.cc:1495] Processing CreateTablet for tablet 168cb98bb1b242a5a0377bf292204d96 (DEFAULT_TABLE table=default.Table_3adc64475e544d4e9c19ed6354bdc9be [id=cdf4cd19161e41daa9626fdde8c99699]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.505903   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 168cb98bb1b242a5a0377bf292204d96. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.505889   904 tablet_service.cc:1495] Processing CreateTablet for tablet 168cb98bb1b242a5a0377bf292204d96 (DEFAULT_TABLE table=default.Table_3adc64475e544d4e9c19ed6354bdc9be [id=cdf4cd19161e41daa9626fdde8c99699]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.506909   904 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 168cb98bb1b242a5a0377bf292204d96. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.513249  1181 ts_tablet_manager.cc:1480] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.055s	user 0.003s	sys 0.001s
I20261120 03:00:53.519316   903 tablet_service.cc:1495] Processing CreateTablet for tablet 88b9e9692a874ae993dde4b9e2586a70 (DEFAULT_TABLE table=default.Table_e4c5ff8f28ae4efca67257942a4fcb77 [id=b30488f6d14b4c9ab1566f2376be602a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.520488   903 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 88b9e9692a874ae993dde4b9e2586a70. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.522931   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d27099bc667b4a1d880af7cec2715bb8" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:53.523473   924 raft_consensus.cc:3296] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:53.524385   924 raft_consensus.cc:2062] T d27099bc667b4a1d880af7cec2715bb8 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:53.526150  1181 tablet_bootstrap.cc:492] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:53.528127   750 tablet_service.cc:1495] Processing CreateTablet for tablet 88b9e9692a874ae993dde4b9e2586a70 (DEFAULT_TABLE table=default.Table_e4c5ff8f28ae4efca67257942a4fcb77 [id=b30488f6d14b4c9ab1566f2376be602a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.529299   750 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 88b9e9692a874ae993dde4b9e2586a70. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.535776  1050 tablet_service.cc:1495] Processing CreateTablet for tablet a6ae8007455d4be3b50672b9c6b0fd8f (DEFAULT_TABLE table=default.Table_f73e1375172b4e9999e9906c1491c0d5 [id=96692554644a476d87e6bdcaa3bd847c]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.537660  1185 raft_consensus.cc:2440] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:53.538586  1185 raft_consensus.cc:2469] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:53.540522   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9882c9cbfbd640a49e3e3c8904ffc1e1" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:53.541211   776 raft_consensus.cc:2062] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:53.542706   857 leader_election.cc:304] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:53.544196  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9882c9cbfbd640a49e3e3c8904ffc1e1" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:53.542045  1185 ts_tablet_manager.cc:1962] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:53.553810   751 tablet_service.cc:1495] Processing CreateTablet for tablet a6ae8007455d4be3b50672b9c6b0fd8f (DEFAULT_TABLE table=default.Table_f73e1375172b4e9999e9906c1491c0d5 [id=96692554644a476d87e6bdcaa3bd847c]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.554975   751 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a6ae8007455d4be3b50672b9c6b0fd8f. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.562493  1181 tablet_bootstrap.cc:654] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:53.565546  1280 raft_consensus.cc:3024] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:53.566390   710 leader_election.cc:304] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:53.568660  1405 raft_consensus.cc:3024] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:53.570048  1405 raft_consensus.cc:744] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:53.571064  1405 consensus_queue.cc:238] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.569317  1050 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a6ae8007455d4be3b50672b9c6b0fd8f. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.582909  1290 raft_consensus.cc:534] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:53.583350  1290 raft_consensus.cc:556] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.585318  1290 leader_election.cc:290] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
W20261120 03:00:53.588472  1011 leader_election.cc:343] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: 9a31431495274c648625b75900b4ac47
W20261120 03:00:53.589576   709 consensus_peers.cc:597] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:53.596867  1045 tablet_service.cc:1495] Processing CreateTablet for tablet 5c5c0ac869a1419fad9f61ec647027ee (DEFAULT_TABLE table=default.Table_5c5911c8746d4b26843ec77c8141d3b6 [id=ffb2040a2e114893855d4d8c66dfa855]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.598860   756 tablet_service.cc:1495] Processing CreateTablet for tablet 5c5c0ac869a1419fad9f61ec647027ee (DEFAULT_TABLE table=default.Table_5c5911c8746d4b26843ec77c8141d3b6 [id=ffb2040a2e114893855d4d8c66dfa855]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.599910   756 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5c5c0ac869a1419fad9f61ec647027ee. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.599663  1048 tablet_service.cc:1495] Processing CreateTablet for tablet 88b9e9692a874ae993dde4b9e2586a70 (DEFAULT_TABLE table=default.Table_e4c5ff8f28ae4efca67257942a4fcb77 [id=b30488f6d14b4c9ab1566f2376be602a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.597980  1045 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5c5c0ac869a1419fad9f61ec647027ee. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.587050   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9a31431495274c648625b75900b4ac47" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:53.598438   898 tablet_service.cc:1495] Processing CreateTablet for tablet 5c5c0ac869a1419fad9f61ec647027ee (DEFAULT_TABLE table=default.Table_5c5911c8746d4b26843ec77c8141d3b6 [id=ffb2040a2e114893855d4d8c66dfa855]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.608126   898 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5c5c0ac869a1419fad9f61ec647027ee. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.611425  1012 leader_election.cc:304] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
W20261120 03:00:53.616302   500 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet bf5a19fc3b434a508e3402e3b1648654 has no consensus state
I20261120 03:00:53.617208   500 master_service.cc:965] ReplaceTablet: received request to replace tablet ce612f9a121b4b18908724d2726e156f from {username='slave'} at 127.0.0.1:45640
I20261120 03:00:53.617890  1290 raft_consensus.cc:2966] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:53.620309   501 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 020be7f7cda0467aab56f657cbe8d0ad
I20261120 03:00:53.621596   501 catalog_manager.cc:6952] ReplaceTablet: tablet 020be7f7cda0467aab56f657cbe8d0ad deleted and replaced by tablet 600a18b7e5104a78935d906b8adfde6f
I20261120 03:00:53.622380  1046 tablet_service.cc:1542] Processing DeleteTablet for tablet 020be7f7cda0467aab56f657cbe8d0ad with delete_type TABLET_DATA_DELETED (replaced by tablet 600a18b7e5104a78935d906b8adfde6f) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:53.624923   899 tablet_service.cc:1542] Processing DeleteTablet for tablet 020be7f7cda0467aab56f657cbe8d0ad with delete_type TABLET_DATA_DELETED (replaced by tablet 600a18b7e5104a78935d906b8adfde6f) from {username='slave'} at 127.0.0.1:44896
==================
WARNING: ThreadSanitizer: data race (pid=977)
  Write of size 8 at 0x7b44000b4e40 by thread T127:
    #0 operator delete(void*) /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 (kudu+0x321819)
    #1 std::__1::_DeallocateCaller::__do_call(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:334:12 (kudu+0x3287c9)
    #2 std::__1::_DeallocateCaller::__do_deallocate_handle_size(void*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:292:12 (kudu+0x328769)
    #3 std::__1::_DeallocateCaller::__do_deallocate_handle_size_align(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:268:14 (libkudu_util.so+0x2aebd2)
    #4 std::__1::__libcpp_deallocate(void*, unsigned long, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/new:340:3 (libkudu_util.so+0x2aeb69)
    #5 std::__1::allocator<std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> > >::deallocate(std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >*, unsigned long) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1798:10 (libkudu_util.so+0x381bcd)
    #6 std::__1::__shared_ptr_emplace<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler, std::__1::allocator<std::__1::shared_ptr<kudu::MemTracker> enable_make_shared<kudu::MemTracker>::make_shared<long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&>(long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<kudu::MemTracker>&)::make_shared_enabler> >::__on_zero_shared_weak() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3515:9 (libkudu_util.so+0x381500)
    #7 std::__1::__shared_weak_count::__release_weak() /home/jenkins-slave/workspace/build_and_test@2/thirdparty/src/llvm-11.0.0.src/projects/libcxx/src/memory.cpp (libc++.so.1+0xbde46)
    #8 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:3384:9 (libkudu_cli_actions.so+0x24d1d5)
    #9 std::__1::shared_ptr<kudu::MemTracker>::~shared_ptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:4098:19 (libkudu_cli_actions.so+0x259ed8)
    #10 kudu::tablet::OpTracker::~OpTracker() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:142:1 (libtablet.so+0x450864)
    #11 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:202:1 (libtablet.so+0x403c45)
    #12 kudu::tablet::TabletReplica::~TabletReplica() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:197:33 (libtablet.so+0x403e59)
    #13 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::DeleteInternal(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:153:44 (libkudu_cli_actions.so+0x2fbea7)
    #14 kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica>::Destruct(kudu::tablet::TabletReplica const*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:116:5 (libkudu_cli_actions.so+0x2fbe69)
    #15 kudu::RefCountedThreadSafe<kudu::tablet::TabletReplica, kudu::DefaultRefCountedThreadSafeTraits<kudu::tablet::TabletReplica> >::Release() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:144:7 (libkudu_cli_actions.so+0x2fbe29)
    #16 scoped_refptr<kudu::tablet::TabletReplica>::~scoped_refptr() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/gutil/ref_counted.h:266:13 (libkudu_cli_actions.so+0x2fa1fa)
    #17 std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >::destroy(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1811:92 (libmaster.so+0x3e1f09)
    #18 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::integral_constant<bool, true>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1703:21 (libmaster.so+0x3e1ee0)
    #19 void std::__1::allocator_traits<std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::destroy<scoped_refptr<kudu::tablet::TabletReplica> >(std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> >&, scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/memory:1544:14 (libmaster.so+0x3e1ea0)
    #20 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::__destruct_at_end(scoped_refptr<kudu::tablet::TabletReplica>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:428:9 (libtserver.so+0x2b4aa6)
    #21 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::clear() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:371:29 (libtserver.so+0x2b49c4)
    #22 std::__1::__vector_base<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~__vector_base() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:465:9 (libtserver.so+0x2b477b)
    #23 std::__1::vector<scoped_refptr<kudu::tablet::TabletReplica>, std::__1::allocator<scoped_refptr<kudu::tablet::TabletReplica> > >::~vector() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:557:5 (libtserver.so+0x28c761)
    #24 kudu::tserver::TSTabletManager::PopulateIncrementalTabletReport(kudu::master::TabletReportPB*, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, bool) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/ts_tablet_manager.cc:1893:1 (libtserver.so+0x2c1b50)
    #25 kudu::tserver::Heartbeater::Thread::GenerateIncrementalTabletReport(kudu::master::TabletReportPB*, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:790:30 (libtserver.so+0x1cb58a)
    #26 kudu::tserver::Heartbeater::Thread::DoHeartbeat(kudu::master::MasterErrorPB*, kudu::rpc::ErrorStatusPB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:521:5 (libtserver.so+0x1cd4c4)
    #27 kudu::tserver::Heartbeater::Thread::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:645:21 (libtserver.so+0x1ce790)
    #28 kudu::tserver::Heartbeater::Thread::Start()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:728:48 (libtserver.so+0x1d03c1)
    #29 decltype(std::__1::forward<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(fp)()) std::__1::__invoke<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtserver.so+0x1d0379)
    #30 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tserver::Heartbeater::Thread::Start()::$_2&>(kudu::tserver::Heartbeater::Thread::Start()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtserver.so+0x1d0309)
    #31 std::__1::__function::__alloc_func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtserver.so+0x1d02d1)
    #32 std::__1::__function::__func<kudu::tserver::Heartbeater::Thread::Start()::$_2, std::__1::allocator<kudu::tserver::Heartbeater::Thread::Start()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtserver.so+0x1cf5cd)
    #33 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #34 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #35 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Previous read of size 8 at 0x7b44000b4e40 by thread T144 (mutexes: write M299906313465758112):
    #0 std::__1::vector<kudu::MemTracker*, std::__1::allocator<kudu::MemTracker*> >::begin() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/vector:1520:30 (libkudu_util.so+0x37b4f9)
    #1 kudu::MemTracker::Release(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/mem_tracker.cc:284:22 (libkudu_util.so+0x378c0a)
    #2 kudu::tablet::OpTracker::Release(kudu::tablet::OpDriver*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_tracker.cc:239:19 (libtablet.so+0x451230)
    #3 kudu::tablet::OpDriver::Finalize() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:608:16 (libtablet.so+0x44a1c2)
    #4 kudu::tablet::OpDriver::ApplyTask() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:579:5 (libtablet.so+0x449953)
    #5 kudu::tablet::OpDriver::ApplyAsync()::$_2::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:65 (libtablet.so+0x44db61)
    #6 decltype(std::__1::forward<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(fp)()) std::__1::__invoke<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x44db19)
    #7 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::OpDriver::ApplyAsync()::$_2&>(kudu::tablet::OpDriver::ApplyAsync()::$_2&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x44daa9)
    #8 std::__1::__function::__alloc_func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x44da71)
    #9 std::__1::__function::__func<kudu::tablet::OpDriver::ApplyAsync()::$_2, std::__1::allocator<kudu::tablet::OpDriver::ApplyAsync()::$_2>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x44cc9d)
    #10 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #11 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #12 kudu::ThreadPool::DispatchThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:733:7 (libkudu_util.so+0x4403a5)
    #13 kudu::ThreadPool::CreateThread()::$_1::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:815:48 (libkudu_util.so+0x442711)
    #14 decltype(std::__1::forward<kudu::ThreadPool::CreateThread()::$_1&>(fp)()) std::__1::__invoke<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_util.so+0x4426c9)
    #15 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::ThreadPool::CreateThread()::$_1&>(kudu::ThreadPool::CreateThread()::$_1&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkudu_util.so+0x442659)
    #16 std::__1::__function::__alloc_func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_util.so+0x442621)
    #17 std::__1::__function::__func<kudu::ThreadPool::CreateThread()::$_1, std::__1::allocator<kudu::ThreadPool::CreateThread()::$_1>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_util.so+0x44191d)
    #18 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #19 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #20 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)

  Mutex M299906313465758112 is already destroyed.

  Thread T127 'heartbeat-1124' (tid=1124, running) created by main thread at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::tserver::Heartbeater::Thread::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:727:10 (libtserver.so+0x1cae61)
    #4 kudu::tserver::Heartbeater::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/heartbeater.cc:251:41 (libtserver.so+0x1cac3d)
    #5 kudu::tserver::TabletServer::Start() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server.cc:167:3 (libtserver.so+0x259ccc)
    #6 kudu::tserver::RunTabletServer() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_server_runner.cc:86:3 (libtserver.so+0x266e93)
    #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:128:10 (libkudu_cli_actions.so+0x3b6429)
    #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkudu_cli_actions.so+0x28c051)
    #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (libkudu_cli_actions.so+0x28bfbc)
    #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkudu_cli_actions.so+0x28bf2c)
    #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkudu_cli_actions.so+0x28ab95)
    #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x14082f)
    #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132da1)
    #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:270:10 (libkudu_tools_util.so+0x130a52)
    #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:114:15 (kudu+0x324910)
    #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:186:16 (kudu+0x325247)
    #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:249:10 (kudu+0x325671)

  Thread T144 'apply [worker]-' (tid=1286, finished) created by thread T82 at:
    #0 pthread_create /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x27b895)
    #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:644:15 (libkudu_util.so+0x423078)
    #2 kudu::Thread::Create(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::function<void ()>, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.h:147:12 (libkudu_cli_actions.so+0x2f29e9)
    #3 kudu::ThreadPool::CreateThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:814:10 (libkudu_util.so+0x43f5cb)
    #4 kudu::ThreadPool::DoSubmit(std::__1::function<void ()>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:599:21 (libkudu_util.so+0x43d8cb)
    #5 kudu::ThreadPool::Submit(std::__1::function<void ()>) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/threadpool.cc:491:10 (libkudu_util.so+0x43fb32)
    #6 kudu::tablet::OpDriver::ApplyAsync() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:526:23 (libtablet.so+0x448cf0)
    #7 kudu::tablet::OpDriver::ReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/ops/op_driver.cc:476:5 (libtablet.so+0x449289)
    #8 kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tablet/tablet_replica.cc:860:15 (libtablet.so+0x41362a)
    #9 decltype(std::__1::forward<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&>(fp)(std::__1::forward<kudu::Status const&>(fp0))) std::__1::__invoke<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libtablet.so+0x4135b8)
    #10 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&>(kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7&, kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libtablet.so+0x413538)
    #11 std::__1::__function::__alloc_func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libtablet.so+0x4134f3)
    #12 std::__1::__function::__func<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7, std::__1::allocator<kudu::tablet::TabletReplica::StartFollowerOp(scoped_refptr<kudu::consensus::ConsensusRound> const&)::$_7>, void (kudu::Status const&)>::operator()(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libtablet.so+0x41270c)
    #13 std::__1::__function::__value_func<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libmaster.so+0x40fe06)
    #14 std::__1::function<void (kudu::Status const&)>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libmaster.so+0x40a5b8)
    #15 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:3598:3 (libconsensus.so+0x1fd9c0)
    #16 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/pending_rounds.cc:200:12 (libconsensus.so+0x1d2e9d)
    #17 kudu::consensus::RaftConsensus::UpdateReplica(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::consensus::RaftConsensus::FlushContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1603:5 (libconsensus.so+0x1f0713)
    #18 kudu::consensus::RaftConsensus::Update(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/raft_consensus.cc:1162:9 (libconsensus.so+0x1efd04)
    #19 kudu::tserver::ConsensusServiceImpl::UpdateConsensus(kudu::consensus::ConsensusRequestPB const*, kudu::consensus::ConsensusResponsePB*, kudu::rpc::RpcContext*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tserver/tablet_service.cc:1792:25 (libtserver.so+0x26e199)
    #20 kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/build/tsan/src/kudu/consensus/consensus.service.cc:321:13 (libconsensus_proto.so+0xd8d34)
    #21 decltype(std::__1::forward<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&>(fp)(std::__1::forward<google::protobuf::Message const*>(fp0), std::__1::forward<google::protobuf::Message*>(fp0), std::__1::forward<kudu::rpc::RpcContext*>(fp0))) std::__1::__invoke<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libconsensus_proto.so+0xd8cc2)
    #22 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*>(kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libconsensus_proto.so+0xd8bf1)
    #23 std::__1::__function::__alloc_func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libconsensus_proto.so+0xd8b6c)
    #24 std::__1::__function::__func<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1, std::__1::allocator<kudu::consensus::ConsensusServiceIf::ConsensusServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_1>, void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libconsensus_proto.so+0xd7e22)
    #25 std::__1::__function::__value_func<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkrpc.so+0x21b0bc)
    #26 std::__1::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkrpc.so+0x21a4f6)
    #27 kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_if.cc:137:3 (libkrpc.so+0x219e9f)
    #28 kudu::rpc::ServicePool::RunThread() /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:228:15 (libkrpc.so+0x21d16f)
    #29 kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/service_pool.cc:91:5 (libkrpc.so+0x21e4c1)
    #30 decltype(std::__1::forward<kudu::rpc::ServicePool::Init(int)::$_0&>(fp)()) std::__1::__invoke<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (libkrpc.so+0x21e479)
    #31 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::rpc::ServicePool::Init(int)::$_0&>(kudu::rpc::ServicePool::Init(int)::$_0&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libkrpc.so+0x21e409)
    #32 std::__1::__function::__alloc_func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libkrpc.so+0x21e3d1)
    #33 std::__1::__function::__func<kudu::rpc::ServicePool::Init(int)::$_0, std::__1::allocator<kudu::rpc::ServicePool::Init(int)::$_0>, void ()>::operator()() /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libkrpc.so+0x21d6cd)
I20261120 03:00:53.640141  1046 tablet_service.cc:1495] Processing CreateTablet for tablet a08aa5ddcf1a4061a70275a8eecb1856 (DEFAULT_TABLE table=default.Table_3c79e29085ad4875874397eaefcab250 [id=47e54c07e54d40adb2ac6907cb03bb68]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.641237  1046 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a08aa5ddcf1a4061a70275a8eecb1856. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.642251  1280 raft_consensus.cc:534] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:53.642604  1280 raft_consensus.cc:3296] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:53.653661  1280 raft_consensus.cc:556] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.655057   899 tablet_service.cc:1495] Processing CreateTablet for tablet a08aa5ddcf1a4061a70275a8eecb1856 (DEFAULT_TABLE table=default.Table_3c79e29085ad4875874397eaefcab250 [id=47e54c07e54d40adb2ac6907cb03bb68]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.656447   899 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a08aa5ddcf1a4061a70275a8eecb1856. 1 dirs total, 0 dirs full, 0 dirs failed
    #34 std::__1::__function::__value_func<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (kudu+0x327654)
    #35 std::__1::function<void ()>::operator()() const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (kudu+0x327479)
    #36 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:695:3 (libkudu_util.so+0x4237da)
I20261120 03:00:53.605322  1405 consensus_queue.cc:1049] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s

SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:126 in operator delete(void*)
I20261120 03:00:53.623876   757 tablet_service.cc:1542] Processing DeleteTablet for tablet 020be7f7cda0467aab56f657cbe8d0ad with delete_type TABLET_DATA_DELETED (replaced by tablet 600a18b7e5104a78935d906b8adfde6f) from {username='slave'} at 127.0.0.1:52644
==================
I20261120 03:00:53.647845   754 tablet_service.cc:1495] Processing CreateTablet for tablet a08aa5ddcf1a4061a70275a8eecb1856 (DEFAULT_TABLE table=default.Table_3c79e29085ad4875874397eaefcab250 [id=47e54c07e54d40adb2ac6907cb03bb68]), partition=RANGE (key) PARTITION UNBOUNDED
W20261120 03:00:53.661507  1079 thread.cc:642] prepare [worker] (thread pool) Time spent creating pthread: real 0.563s	user 0.011s	sys 0.017s
W20261120 03:00:53.661804  1079 thread.cc:609] prepare [worker] (thread pool) Time spent starting thread: real 0.564s	user 0.011s	sys 0.017s
W20261120 03:00:53.662487   489 catalog_manager.cc:4863] Async tablet task AlterTable RPC for tablet bf5a19fc3b434a508e3402e3b1648654 (table default.Table_a925e1ca8a3d4ee18ad6481a8e3c1a73 [id=7b983cbb686c4cffa5eb82d5db6b5682], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet bf5a19fc3b434a508e3402e3b1648654 has no consensus state
I20261120 03:00:53.663190  1392 ts_tablet_manager.cc:1962] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:53.672865  1382 tablet.cc:1872] T 1237b0d2fa414ef38271835b5d69d6b9 P 823abdbd90ba47bbacad0f8dcff78234: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
W20261120 03:00:53.598541   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 56001.4us
I20261120 03:00:53.603120  1180 tablet_bootstrap.cc:654] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:53.604781   900 consensus_meta.cc:298] T 0db7db25d779459ebde532f938493975Time spent flushing consensus metadata: real 0.571s	user 0.007s	sys 0.003s
I20261120 03:00:53.688989  1324 tablet_bootstrap.cc:492] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:53.689399  1324 ts_tablet_manager.cc:1449] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.328s	user 0.007s	sys 0.005s
I20261120 03:00:53.691774  1324 raft_consensus.cc:366] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.692391  1324 raft_consensus.cc:392] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:53.692629  1324 raft_consensus.cc:787] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:53.694356  1048 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 88b9e9692a874ae993dde4b9e2586a70. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.696023  1181 tablet_bootstrap.cc:492] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:53.696553  1181 ts_tablet_manager.cc:1449] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.171s	user 0.011s	sys 0.000s
I20261120 03:00:53.699160  1181 raft_consensus.cc:366] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.700009  1181 raft_consensus.cc:392] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:53.700347  1181 raft_consensus.cc:787] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:53.701239  1181 consensus_queue.cc:261] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.707605  1181 ts_tablet_manager.cc:1480] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.011s	user 0.005s	sys 0.000s
I20261120 03:00:53.716128   754 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a08aa5ddcf1a4061a70275a8eecb1856. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.728960  1181 tablet_bootstrap.cc:492] T fa57d0c282964e37b9133ea57f458a51 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:53.746475   501 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 1237b0d2fa414ef38271835b5d69d6b9 (table default.Table_523caa128be441e1a52767e3d3318ce0 [id=93a0027d82064a208e2535349e9b042d]). Expected version 1 got 0
I20261120 03:00:53.749629  1045 tablet_service.cc:1495] Processing CreateTablet for tablet c6fea104598e4f6d876f96675f2da816 (DEFAULT_TABLE table=default.Table_3d732539a5434ae193267b36578eb1c3 [id=376c8b210df847dda73d22051971b065]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.752719   751 tablet_service.cc:1495] Processing CreateTablet for tablet c6fea104598e4f6d876f96675f2da816 (DEFAULT_TABLE table=default.Table_3d732539a5434ae193267b36578eb1c3 [id=376c8b210df847dda73d22051971b065]), partition=RANGE (key) PARTITION UNBOUNDED
W20261120 03:00:53.765447   857 leader_election.cc:343] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:53.772828  1050 tablet_service.cc:1495] Processing CreateTablet for tablet 3d89fb1d7fa14ae8ac252c68338ed4d6 (DEFAULT_TABLE table=default.Table_51385081065c4b11955d65846ec63d6f [id=9e242cff711d4d7082745f6861c9724a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.774164  1185 ts_tablet_manager.cc:1975] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:53.774511  1185 log.cc:1199] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/00dd2ffad1124244b171b2657d38a708
I20261120 03:00:53.775120  1185 ts_tablet_manager.cc:1996] T 00dd2ffad1124244b171b2657d38a708 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:53.775861   898 tablet_service.cc:1495] Processing CreateTablet for tablet c6fea104598e4f6d876f96675f2da816 (DEFAULT_TABLE table=default.Table_3d732539a5434ae193267b36578eb1c3 [id=376c8b210df847dda73d22051971b065]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.777133  1324 consensus_queue.cc:261] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.779131  1324 ts_tablet_manager.cc:1480] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.089s	user 0.005s	sys 0.000s
I20261120 03:00:53.780114  1324 tablet_bootstrap.cc:492] T ce612f9a121b4b18908724d2726e156f P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:53.782032   751 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c6fea104598e4f6d876f96675f2da816. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.786324   898 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c6fea104598e4f6d876f96675f2da816. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.790285  1072 raft_consensus.cc:3296] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:53.792044  1324 tablet_bootstrap.cc:654] T ce612f9a121b4b18908724d2726e156f P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:53.762406   900 consensus_meta.cc:340] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440: Time spent flushing consensus metadata: real 0.729s	user 0.008s	sys 0.004s
I20261120 03:00:53.823045   501 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_25c78056a3a44c77bb4bd8bf7a471fe5"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:53.824923   501 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_25c78056a3a44c77bb4bd8bf7a471fe5 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:53.826924  1059 tablet_service.cc:1542] Processing DeleteTablet for tablet 9f73b223756845bb8af7b7fd96c0f9fc with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:53.829361  1047 tablet_service.cc:1542] Processing DeleteTablet for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 with delete_type TABLET_DATA_DELETED (replaced by tablet 68179488b9bc4e03b65a9f1d434c95b5) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:53.830276  1059 tablet_service.cc:1542] Processing DeleteTablet for tablet c3b56d0ccab140c28a37808d9d5902c6 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:53.831224  1047 tablet_service.cc:1542] Processing DeleteTablet for tablet 06c3bac919e342b09dd6af13aebc387b with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:53.848443  1047 tablet_service.cc:1542] Processing DeleteTablet for tablet a642c32c1b6043978516d97efa2347a3 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:53.855518  1047 tablet_service.cc:1542] Processing DeleteTablet for tablet a72d5d7896de4020a1ac4d2f8c551e15 with delete_type TABLET_DATA_DELETED (replaced by tablet ce612f9a121b4b18908724d2726e156f) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:53.858778  1047 tablet_service.cc:1542] Processing DeleteTablet for tablet d753dd170cab4f18980e85707c4093e9 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:53.866245   901 tablet_service.cc:1495] Processing CreateTablet for tablet a6ae8007455d4be3b50672b9c6b0fd8f (DEFAULT_TABLE table=default.Table_f73e1375172b4e9999e9906c1491c0d5 [id=96692554644a476d87e6bdcaa3bd847c]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.867081  1180 tablet_bootstrap.cc:492] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:53.867522  1180 ts_tablet_manager.cc:1449] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.372s	user 0.010s	sys 0.007s
I20261120 03:00:53.867630  1280 leader_election.cc:290] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:53.870492   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9882c9cbfbd640a49e3e3c8904ffc1e1" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:53.871101   776 raft_consensus.cc:3296] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:53.872020   776 raft_consensus.cc:2062] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:00:53.876900  1045 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c6fea104598e4f6d876f96675f2da816. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.879163  1050 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3d89fb1d7fa14ae8ac252c68338ed4d6. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.885484  1180 raft_consensus.cc:366] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:53.889676  1180 raft_consensus.cc:392] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:53.889971  1180 raft_consensus.cc:787] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:53.893479  1079 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9882c9cbfbd640a49e3e3c8904ffc1e1" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:53.881215  1295 ts_tablet_manager.cc:1975] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:53.894385  1295 log.cc:1199] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/0fd4a37075a34ec6bb0d6eb950045a1b
I20261120 03:00:53.895246  1295 ts_tablet_manager.cc:1996] T 0fd4a37075a34ec6bb0d6eb950045a1b P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
W20261120 03:00:53.895645   857 leader_election.cc:343] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:53.895767  1324 tablet_bootstrap.cc:492] T ce612f9a121b4b18908724d2726e156f P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:53.896158  1324 ts_tablet_manager.cc:1449] T ce612f9a121b4b18908724d2726e156f P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.116s	user 0.026s	sys 0.021s
I20261120 03:00:53.898525  1324 raft_consensus.cc:366] T ce612f9a121b4b18908724d2726e156f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:53.899123  1324 raft_consensus.cc:392] T ce612f9a121b4b18908724d2726e156f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:53.899384  1324 raft_consensus.cc:787] T ce612f9a121b4b18908724d2726e156f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:53.900064  1324 consensus_queue.cc:261] T ce612f9a121b4b18908724d2726e156f P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:53.906015  1324 ts_tablet_manager.cc:1480] T ce612f9a121b4b18908724d2726e156f P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.010s	user 0.005s	sys 0.001s
I20261120 03:00:53.906900  1324 tablet_bootstrap.cc:492] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:53.912279  1324 tablet_bootstrap.cc:654] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:53.928222  1324 tablet_bootstrap.cc:492] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:53.928622  1324 ts_tablet_manager.cc:1449] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.022s	user 0.008s	sys 0.000s
I20261120 03:00:53.930476   901 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a6ae8007455d4be3b50672b9c6b0fd8f. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.931026  1324 raft_consensus.cc:366] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:53.931596  1324 raft_consensus.cc:392] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:53.931831  1324 raft_consensus.cc:787] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:53.936174   902 tablet_service.cc:1495] Processing CreateTablet for tablet 3d89fb1d7fa14ae8ac252c68338ed4d6 (DEFAULT_TABLE table=default.Table_51385081065c4b11955d65846ec63d6f [id=9e242cff711d4d7082745f6861c9724a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:53.937444   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3d89fb1d7fa14ae8ac252c68338ed4d6. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.944708   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 00dd2ffad1124244b171b2657d38a708 (table default.Table_e07b2ba4d9864d148d7088622b10e03c [id=201bf626c7d643d7a98a680b1e714ba3]) successfully deleted
W20261120 03:00:53.945349   486 catalog_manager.cc:5061] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 1885397e42774a89bb347bc776284bc0 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 1885397e42774a89bb347bc776284bc0 already in progress: creating tablet
I20261120 03:00:53.945372   754 tablet_service.cc:1495] Processing CreateTablet for tablet 3d89fb1d7fa14ae8ac252c68338ed4d6 (DEFAULT_TABLE table=default.Table_51385081065c4b11955d65846ec63d6f [id=9e242cff711d4d7082745f6861c9724a]), partition=RANGE (key) PARTITION UNBOUNDED
W20261120 03:00:53.945937   486 catalog_manager.cc:5061] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 020be7f7cda0467aab56f657cbe8d0ad because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 020be7f7cda0467aab56f657cbe8d0ad already in progress: creating tablet
I20261120 03:00:53.946349   754 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3d89fb1d7fa14ae8ac252c68338ed4d6. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:53.853981  1181 tablet_bootstrap.cc:654] T fa57d0c282964e37b9133ea57f458a51 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:53.951814   494 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet b6f0ff2246cd4772af47812b699e93e8 consensus state has no leader
I20261120 03:00:53.953881   493 master_service.cc:965] ReplaceTablet: received request to replace tablet 88b9e9692a874ae993dde4b9e2586a70 from {username='slave'} at 127.0.0.1:45640
I20261120 03:00:53.942342  1324 consensus_queue.cc:261] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:53.960860  1324 ts_tablet_manager.cc:1480] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.032s	user 0.006s	sys 0.000s
I20261120 03:00:53.961783  1324 tablet_bootstrap.cc:492] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:53.967386  1324 tablet_bootstrap.cc:654] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:53.969527   499 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.65:40295 (request call id 29) took 855 ms (client timeout 1000 ms). Trace:
W20261120 03:00:53.969873   499 rpcz_store.cc:269] 1120 03:00:53.114212 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.114872 (+   660us) service_pool.cc:224] Handling call
1120 03:00:53.969466 (+854594us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:53.546290 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:53.546683 (+   393us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:53.546705 (+    22us) write_op.cc:440] Acquired schema lock
1120 03:00:53.546718 (+    13us) tablet.cc:710] Decoding operations
1120 03:00:53.547046 (+   328us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:53.547080 (+    34us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:53.547100 (+    20us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:00:53.547397 (+   297us) tablet.cc:758] Row locks acquired
1120 03:00:53.547408 (+    11us) write_op.cc:260] PREPARE: finished
1120 03:00:53.547528 (+   120us) write_op.cc:270] Start()
1120 03:00:53.547613 (+    85us) write_op.cc:276] Timestamp: P: 1795143653547452 usec, L: 0
1120 03:00:53.547626 (+    13us) op_driver.cc:350] REPLICATION: starting
1120 03:00:53.548125 (+   499us) log.cc:844] Serialized 901 byte log entry
1120 03:00:53.945368 (+397243us) op_driver.cc:466] REPLICATION: finished
1120 03:00:53.950237 (+  4869us) write_op.cc:301] APPLY: starting
1120 03:00:53.950294 (+    57us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:53.950397 (+   103us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:53.950493 (+    96us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:53.950700 (+   207us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:53.950836 (+   136us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
1120 03:00:53.950874 (+    38us) write_op.cc:312] APPLY: finished
1120 03:00:53.951036 (+   162us) log.cc:844] Serialized 29 byte log entry
1120 03:00:53.959281 (+  8245us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:53.959363 (+    82us) write_op.cc:459] Released schema lock
1120 03:00:53.959399 (+    36us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"reported_tablets":5,"child_traces":[["op",{"apply.queue_time_us":4712,"apply.run_cpu_time_us":1369,"apply.run_wall_time_us":9389,"mutex_wait_us":8036,"num_ops":2,"prepare.queue_time_us":277359,"prepare.run_cpu_time_us":2033,"prepare.run_wall_time_us":2034,"replication_time_us":397660,"threads_started":1}]]}
I20261120 03:00:53.974191  1295 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 66) took 1631 ms. Trace:
I20261120 03:00:53.974522  1295 rpcz_store.cc:276] 1120 03:00:52.342556 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:52.343163 (+   607us) service_pool.cc:224] Handling call
1120 03:00:53.404437 (+1061274us) ts_tablet_manager.cc:1197] Deleting tablet 0fd4a37075a34ec6bb0d6eb950045a1b
1120 03:00:53.404506 (+    69us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:53.776990 (+372484us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:53.881089 (+104099us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:53.974103 (+ 93014us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1060668}
I20261120 03:00:53.974134   496 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 9882c9cbfbd640a49e3e3c8904ffc1e1 (table default.Table_0879bd445c234e4faefff97cd088035d [id=53fb1849755c4debad2dbfbf37271fbd]). Expected version 1 got 0
I20261120 03:00:53.978063  1392 ts_tablet_manager.cc:1975] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.2
I20261120 03:00:53.978494  1392 log.cc:1199] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/5f5c8711a6eb43c3aff1c27ed5115d6a
I20261120 03:00:53.979207  1392 ts_tablet_manager.cc:1996] T 5f5c8711a6eb43c3aff1c27ed5115d6a P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:53.980821   857 leader_election.cc:304] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:53.981215  1180 consensus_queue.cc:261] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:53.982216  1280 raft_consensus.cc:3024] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:53.982945   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 0fd4a37075a34ec6bb0d6eb950045a1b (table default.Table_d52207672f8f40acb7026fec3eb73d85 [id=563f250386d546eb8b53db93e8139e9b]) successfully deleted
I20261120 03:00:53.982892  1280 raft_consensus.cc:744] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:00:53.983994  1280 consensus_queue.cc:238] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:53.985496  1295 tablet_replica.cc:333] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:54.031874  1324 tablet_bootstrap.cc:492] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:54.032269  1324 ts_tablet_manager.cc:1449] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.071s	user 0.008s	sys 0.003s
I20261120 03:00:54.036607  1181 tablet_bootstrap.cc:492] T fa57d0c282964e37b9133ea57f458a51 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:54.037055  1181 ts_tablet_manager.cc:1449] T fa57d0c282964e37b9133ea57f458a51 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.308s	user 0.009s	sys 0.000s
I20261120 03:00:54.044611  1295 raft_consensus.cc:2440] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:54.044814  1324 raft_consensus.cc:366] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:54.045295  1180 ts_tablet_manager.cc:1480] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.177s	user 0.006s	sys 0.002s
I20261120 03:00:54.045416  1324 raft_consensus.cc:392] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.045670  1324 raft_consensus.cc:787] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.044329  1466 tablet.cc:1872] T 5fee8dff291c43e9be0fb10b308718ce P 3d30c2428730481db9b0ffd279dd3440: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20261120 03:00:54.046300  1324 consensus_queue.cc:261] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:54.047652  1295 raft_consensus.cc:2469] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:54.048460   754 tablet_service.cc:1542] Processing DeleteTablet for tablet 00dd2ffad1124244b171b2657d38a708 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:52 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:54.049716   754 tablet_service.cc:1542] Processing DeleteTablet for tablet 200b234da897419a82bf5844620adf24 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:52 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:54.055538  1181 raft_consensus.cc:366] T fa57d0c282964e37b9133ea57f458a51 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:54.056226  1181 raft_consensus.cc:392] T fa57d0c282964e37b9133ea57f458a51 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.056479  1181 raft_consensus.cc:787] T fa57d0c282964e37b9133ea57f458a51 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.058990  1324 ts_tablet_manager.cc:1480] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.026s	user 0.006s	sys 0.000s
I20261120 03:00:54.068147  1324 tablet_bootstrap.cc:492] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:54.066639  1465 tablet.cc:1872] T 5fee8dff291c43e9be0fb10b308718ce P 5beeefe8bef54caeac463d05d8c30fa6: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20261120 03:00:54.069962   494 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_f601bffe6ddb49b3a3c589ac7b25235c" } new_table_name: "default.Table_d2bd3060da1545548f422436c9c7c22e" modify_external_catalogs: true
I20261120 03:00:54.074414  1437 consensus_queue.cc:1049] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20261120 03:00:54.075042   498 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 5c5c0ac869a1419fad9f61ec647027ee
I20261120 03:00:54.046432  1180 tablet_bootstrap.cc:492] T fbbe19dadb08455285078e239d81bcca P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:54.077653  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 46) took 1765 ms. Trace:
I20261120 03:00:54.077997  1392 rpcz_store.cc:276] 1120 03:00:52.313191 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:52.313554 (+   363us) service_pool.cc:224] Handling call
1120 03:00:53.458598 (+1145044us) ts_tablet_manager.cc:1197] Deleting tablet 5f5c8711a6eb43c3aff1c27ed5115d6a
1120 03:00:53.458663 (+    65us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:53.785452 (+326789us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:53.977909 (+192457us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:54.077587 (+ 99678us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1144301}
I20261120 03:00:54.079609  1392 tablet_replica.cc:333] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:54.080506  1392 raft_consensus.cc:2440] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:54.080960  1392 raft_consensus.cc:2469] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:54.088012  1324 tablet_bootstrap.cc:654] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.088588  1392 ts_tablet_manager.cc:1962] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:54.091540   754 tablet_service.cc:1542] Processing DeleteTablet for tablet 5c5c0ac869a1419fad9f61ec647027ee with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:53 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:54.093307  1050 tablet_service.cc:1542] Processing DeleteTablet for tablet 5c5c0ac869a1419fad9f61ec647027ee with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:53 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:54.097393   899 tablet_service.cc:1542] Processing DeleteTablet for tablet 5c5c0ac869a1419fad9f61ec647027ee with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:53 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:54.098106  1181 consensus_queue.cc:261] T fa57d0c282964e37b9133ea57f458a51 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
W20261120 03:00:54.098999   857 consensus_peers.cc:597] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:54.099895  1295 ts_tablet_manager.cc:1962] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
W20261120 03:00:54.100288   486 catalog_manager.cc:5048] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 00dd2ffad1124244b171b2657d38a708 because the tablet was not found. No further retry: Not found: Tablet not found: 00dd2ffad1124244b171b2657d38a708
W20261120 03:00:54.107621   486 catalog_manager.cc:5048] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 200b234da897419a82bf5844620adf24 because the tablet was not found. No further retry: Not found: Tablet not found: 200b234da897419a82bf5844620adf24
W20261120 03:00:54.115391   486 catalog_manager.cc:5061] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 5c5c0ac869a1419fad9f61ec647027ee because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 5c5c0ac869a1419fad9f61ec647027ee already in progress: creating tablet
I20261120 03:00:54.114605  1181 ts_tablet_manager.cc:1480] T fa57d0c282964e37b9133ea57f458a51 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.077s	user 0.007s	sys 0.000s
I20261120 03:00:54.124250  1181 tablet_bootstrap.cc:492] T 5fee8dff291c43e9be0fb10b308718ce P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:54.124804  1437 raft_consensus.cc:534] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:54.125221  1437 raft_consensus.cc:556] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.127480  1437 leader_election.cc:290] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:54.130136  1072 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "020be7f7cda0467aab56f657cbe8d0ad" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:54.132467   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "020be7f7cda0467aab56f657cbe8d0ad" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
W20261120 03:00:54.134105   857 leader_election.cc:343] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:54.135864   857 leader_election.cc:343] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:54.138222   857 leader_election.cc:304] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:54.140076  1437 raft_consensus.cc:2966] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:54.104439  1180 tablet_bootstrap.cc:654] T fbbe19dadb08455285078e239d81bcca P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.149456  1181 tablet_bootstrap.cc:654] T 5fee8dff291c43e9be0fb10b308718ce P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.159863   497 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_28990e82c07a47a29f4dbc3da17c42bb" } new_table_name: "default.Table_fc294aebad394eb1b7c3ddd38d9f3a51" modify_external_catalogs: true
I20261120 03:00:54.167918  1181 tablet_bootstrap.cc:492] T 5fee8dff291c43e9be0fb10b308718ce P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:54.168313  1181 ts_tablet_manager.cc:1449] T 5fee8dff291c43e9be0fb10b308718ce P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.044s	user 0.007s	sys 0.005s
I20261120 03:00:54.176352  1181 raft_consensus.cc:366] T 5fee8dff291c43e9be0fb10b308718ce P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.176978  1181 raft_consensus.cc:392] T 5fee8dff291c43e9be0fb10b308718ce P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.177249  1181 raft_consensus.cc:787] T 5fee8dff291c43e9be0fb10b308718ce P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.184834  1290 raft_consensus.cc:534] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:54.185289  1290 raft_consensus.cc:556] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:54.193284  1290 leader_election.cc:290] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
W20261120 03:00:54.195827  1012 leader_election.cc:343] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Not found: Tablet not found: df61317717d3401eb74cd93aa336c4dc
W20261120 03:00:54.196753  1011 leader_election.cc:343] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: df61317717d3401eb74cd93aa336c4dc
I20261120 03:00:54.197115  1011 leader_election.cc:304] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:54.198119  1290 raft_consensus.cc:2966] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:54.222473  1324 tablet_bootstrap.cc:492] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:54.222919  1324 ts_tablet_manager.cc:1449] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.155s	user 0.023s	sys 0.017s
I20261120 03:00:54.225274  1324 raft_consensus.cc:366] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.225915  1324 raft_consensus.cc:392] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.226172  1324 raft_consensus.cc:787] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.226771  1324 consensus_queue.cc:261] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.228693  1324 ts_tablet_manager.cc:1480] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.006s	user 0.004s	sys 0.000s
I20261120 03:00:54.177836  1181 consensus_queue.cc:261] T 5fee8dff291c43e9be0fb10b308718ce P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.234042  1181 ts_tablet_manager.cc:1480] T 5fee8dff291c43e9be0fb10b308718ce P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.065s	user 0.007s	sys 0.000s
I20261120 03:00:54.237231  1181 tablet_bootstrap.cc:492] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:54.240984  1437 raft_consensus.cc:534] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:54.241459  1437 raft_consensus.cc:556] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.243455  1437 leader_election.cc:290] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:54.246155  1072 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "daae644853734cd5818a46ecaa5ce7df" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:54.247733   857 leader_election.cc:343] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:54.249590  1181 tablet_bootstrap.cc:654] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.250313   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "daae644853734cd5818a46ecaa5ce7df" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:54.250841  1324 tablet_bootstrap.cc:492] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
W20261120 03:00:54.254029   857 leader_election.cc:343] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:54.254429   857 leader_election.cc:304] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:54.254962  1180 tablet_bootstrap.cc:492] T fbbe19dadb08455285078e239d81bcca P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:54.255409  1180 ts_tablet_manager.cc:1449] T fbbe19dadb08455285078e239d81bcca P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.209s	user 0.010s	sys 0.009s
I20261120 03:00:54.255374  1437 raft_consensus.cc:2966] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:54.259068  1180 raft_consensus.cc:366] T fbbe19dadb08455285078e239d81bcca P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.260145  1180 raft_consensus.cc:392] T fbbe19dadb08455285078e239d81bcca P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.263072  1180 raft_consensus.cc:787] T fbbe19dadb08455285078e239d81bcca P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.261847  1050 tablet_service.cc:1542] Processing DeleteTablet for tablet 5f5c8711a6eb43c3aff1c27ed5115d6a with delete_type TABLET_DATA_DELETED (replaced by tablet a1854e4d9016414ea5c086ad56224b1f) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:54.264968  1050 tablet_service.cc:1542] Processing DeleteTablet for tablet 136056e5242a42ef8fa4b40c989b5423 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:54.267325  1050 tablet_service.cc:1542] Processing DeleteTablet for tablet 9a31431495274c648625b75900b4ac47 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:54.268029  1324 tablet_bootstrap.cc:654] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.265237  1180 consensus_queue.cc:261] T fbbe19dadb08455285078e239d81bcca P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.269672  1050 tablet_service.cc:1542] Processing DeleteTablet for tablet df61317717d3401eb74cd93aa336c4dc with delete_type TABLET_DATA_DELETED (replaced by tablet 88b9e9692a874ae993dde4b9e2586a70) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:54.270737  1295 ts_tablet_manager.cc:1975] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:54.271075  1295 log.cc:1199] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/9a31431495274c648625b75900b4ac47
I20261120 03:00:54.271670  1295 ts_tablet_manager.cc:1996] T 9a31431495274c648625b75900b4ac47 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:54.272351  1392 ts_tablet_manager.cc:1975] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:54.272701  1392 log.cc:1199] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/0fd4a37075a34ec6bb0d6eb950045a1b
I20261120 03:00:54.273311  1392 ts_tablet_manager.cc:1996] T 0fd4a37075a34ec6bb0d6eb950045a1b P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:54.286789  1295 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 67) took 1941 ms. Trace:
I20261120 03:00:54.287137  1295 rpcz_store.cc:276] 1120 03:00:52.345641 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:52.345768 (+   127us) service_pool.cc:224] Handling call
1120 03:00:53.985340 (+1639572us) ts_tablet_manager.cc:1197] Deleting tablet 9a31431495274c648625b75900b4ac47
1120 03:00:53.985392 (+    52us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:54.046905 (+ 61513us) spinlock_profiling.cc:245] Waited 12.7 ms on lock 0x7b54000f0854. stack: 00007f57a3e09023 00007f57a3e08f20 00007f57a39a4574 00007f57ad5c1964 00007f57ad5c18f9 00007f57aaf31a41 00007f57aafd26a2 00007f57ab43286c 00007f57ac09dffa 00007f57ac0d00b0 00007f57ac0b643d 00007f57ac0b63e9 00007f57ac0b6379 00007f57ac0b6341 00007f57ac0b556d 0000000000327654
1120 03:00:54.179940 (+133035us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:54.270598 (+ 90658us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:54.286731 (+ 16133us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":8729,"spinlock_wait_cycles":29149056,"tablet-delete.queue_time_us":1639002}
I20261120 03:00:54.288518  1295 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 68) took 1926 ms. Trace:
I20261120 03:00:54.289896   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 9a31431495274c648625b75900b4ac47 (table default.Table_3bb077af5e15449b85500aca3a1500c3 [id=234b77d1cf1847b0ae8afa8d3dbdff71]) successfully deleted
I20261120 03:00:54.291399  1181 tablet_bootstrap.cc:492] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:54.291807  1181 ts_tablet_manager.cc:1449] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.055s	user 0.011s	sys 0.000s
I20261120 03:00:54.293097  1324 tablet_bootstrap.cc:492] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:54.292788  1295 rpcz_store.cc:276] 1120 03:00:52.362198 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:52.362353 (+   155us) service_pool.cc:224] Handling call
1120 03:00:54.288172 (+1925819us) ts_tablet_manager.cc:1197] Deleting tablet df61317717d3401eb74cd93aa336c4dc
1120 03:00:54.288229 (+    57us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:54.288450 (+   221us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1925206}
I20261120 03:00:54.293566  1324 ts_tablet_manager.cc:1449] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.043s	user 0.008s	sys 0.002s
W20261120 03:00:54.294516   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet df61317717d3401eb74cd93aa336c4dc because the tablet was not found. No further retry: Not found: Tablet not found: df61317717d3401eb74cd93aa336c4dc
I20261120 03:00:54.295437  1295 tablet_replica.cc:333] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:54.296154  1324 raft_consensus.cc:366] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.296797  1324 raft_consensus.cc:392] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.297055  1324 raft_consensus.cc:787] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.296952  1295 raft_consensus.cc:2440] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:54.297739  1295 raft_consensus.cc:2469] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:54.300216  1295 ts_tablet_manager.cc:1962] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:54.305064  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 55) took 1928 ms. Trace:
I20261120 03:00:54.297736  1324 consensus_queue.cc:261] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.305462  1392 rpcz_store.cc:276] 1120 03:00:52.376384 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:52.376488 (+   104us) service_pool.cc:224] Handling call
1120 03:00:54.079457 (+1702969us) ts_tablet_manager.cc:1197] Deleting tablet 0fd4a37075a34ec6bb0d6eb950045a1b
1120 03:00:54.079502 (+    45us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:54.182143 (+102641us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:54.272224 (+ 90081us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:54.304997 (+ 32773us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1701831}
I20261120 03:00:54.306980  1392 tablet_replica.cc:333] T fa57d0c282964e37b9133ea57f458a51 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:54.307781  1392 raft_consensus.cc:2440] T fa57d0c282964e37b9133ea57f458a51 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:54.308139  1324 ts_tablet_manager.cc:1480] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.014s	user 0.003s	sys 0.004s
I20261120 03:00:54.308259  1392 raft_consensus.cc:2469] T fa57d0c282964e37b9133ea57f458a51 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:54.309031  1324 tablet_bootstrap.cc:492] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:54.311236  1392 ts_tablet_manager.cc:1962] T fa57d0c282964e37b9133ea57f458a51 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:54.312852   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 0fd4a37075a34ec6bb0d6eb950045a1b (table default.Table_d52207672f8f40acb7026fec3eb73d85 [id=563f250386d546eb8b53db93e8139e9b]) successfully deleted
I20261120 03:00:54.315668   502 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet a08aa5ddcf1a4061a70275a8eecb1856
I20261120 03:00:54.317129   502 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:45640 (request call id 131) took 1035 ms. Trace:
I20261120 03:00:54.320495   754 tablet_service.cc:1542] Processing DeleteTablet for tablet a08aa5ddcf1a4061a70275a8eecb1856 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:53 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:54.322096  1050 tablet_service.cc:1542] Processing DeleteTablet for tablet a08aa5ddcf1a4061a70275a8eecb1856 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:53 UTC) from {username='slave'} at 127.0.0.1:50744
W20261120 03:00:54.322109   486 catalog_manager.cc:5061] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet a08aa5ddcf1a4061a70275a8eecb1856 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet a08aa5ddcf1a4061a70275a8eecb1856 already in progress: creating tablet
I20261120 03:00:54.325444  1181 raft_consensus.cc:366] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.327771  1181 raft_consensus.cc:392] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.328389  1181 raft_consensus.cc:787] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.317409   502 rpcz_store.cc:276] 1120 03:00:53.282057 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.282199 (+   142us) service_pool.cc:224] Handling call
1120 03:00:53.288329 (+  6130us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:00:53.288859 (+   530us) catalog_manager.cc:2859] Modifying in-memory table state
1120 03:00:53.288973 (+   114us) catalog_manager.cc:2868] Locking tablets
1120 03:00:53.289498 (+   525us) catalog_manager.cc:2882] Removing table and tablets from system table
1120 03:00:54.315295 (+1025797us) catalog_manager.cc:2900] Removing table from by-name map
1120 03:00:54.315488 (+   193us) catalog_manager.cc:2916] Committing in-memory state
1120 03:00:54.315583 (+    95us) catalog_manager.cc:2920] Aborting table tasks
1120 03:00:54.317072 (+  1489us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:53.608399 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:53.608796 (+   397us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:53.608818 (+    22us) write_op.cc:440] Acquired schema lock
1120 03:00:53.608851 (+    33us) tablet.cc:710] Decoding operations
1120 03:00:53.609161 (+   310us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:53.609195 (+    34us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:53.609216 (+    21us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:00:53.609509 (+   293us) tablet.cc:758] Row locks acquired
1120 03:00:53.609523 (+    14us) write_op.cc:260] PREPARE: finished
1120 03:00:53.609640 (+   117us) write_op.cc:270] Start()
1120 03:00:53.609722 (+    82us) write_op.cc:276] Timestamp: P: 1795143653609557 usec, L: 0
1120 03:00:53.609750 (+    28us) op_driver.cc:350] REPLICATION: starting
1120 03:00:53.610236 (+   486us) log.cc:844] Serialized 725 byte log entry
1120 03:00:54.213539 (+603303us) op_driver.cc:466] REPLICATION: finished
1120 03:00:54.306211 (+ 92672us) write_op.cc:301] APPLY: starting
1120 03:00:54.306275 (+    64us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:54.306390 (+   115us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:54.306485 (+    95us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:54.306755 (+   270us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:54.306887 (+   132us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
1120 03:00:54.306930 (+    43us) write_op.cc:312] APPLY: finished
1120 03:00:54.314405 (+  7475us) log.cc:844] Serialized 29 byte log entry
1120 03:00:54.314571 (+   166us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:54.314646 (+    75us) write_op.cc:459] Released schema lock
1120 03:00:54.314676 (+    30us) write_op.cc:346] FINISH: Updating metrics
1120 03:00:54.314912 (+   236us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:54.314940 (+    28us) write_op.cc:459] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":92488,"apply.run_cpu_time_us":1430,"apply.run_wall_time_us":8666,"num_ops":2,"prepare.queue_time_us":316960,"prepare.run_cpu_time_us":2016,"prepare.run_wall_time_us":2015,"replication_time_us":603686}]]}
I20261120 03:00:54.329470  1181 consensus_queue.cc:261] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.331537  1324 tablet_bootstrap.cc:654] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.339553   500 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet ce612f9a121b4b18908724d2726e156f
I20261120 03:00:54.340991   500 catalog_manager.cc:6952] ReplaceTablet: tablet ce612f9a121b4b18908724d2726e156f deleted and replaced by tablet 5da3d2724dd34aed88e6b08983188004
I20261120 03:00:54.342469   754 tablet_service.cc:1542] Processing DeleteTablet for tablet ce612f9a121b4b18908724d2726e156f with delete_type TABLET_DATA_DELETED (replaced by tablet 5da3d2724dd34aed88e6b08983188004) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:54.343235  1050 tablet_service.cc:1542] Processing DeleteTablet for tablet ce612f9a121b4b18908724d2726e156f with delete_type TABLET_DATA_DELETED (replaced by tablet 5da3d2724dd34aed88e6b08983188004) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:54.344125  1185 tablet_replica.cc:333] T ce612f9a121b4b18908724d2726e156f P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:54.344112   899 tablet_service.cc:1542] Processing DeleteTablet for tablet ce612f9a121b4b18908724d2726e156f with delete_type TABLET_DATA_DELETED (replaced by tablet 5da3d2724dd34aed88e6b08983188004) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:54.344846  1185 raft_consensus.cc:2440] T ce612f9a121b4b18908724d2726e156f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:54.345839  1185 raft_consensus.cc:2469] T ce612f9a121b4b18908724d2726e156f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:54.352423  1324 tablet_bootstrap.cc:492] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:54.352828  1324 ts_tablet_manager.cc:1449] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.044s	user 0.013s	sys 0.000s
I20261120 03:00:54.353596  1185 ts_tablet_manager.cc:1962] T ce612f9a121b4b18908724d2726e156f P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:54.355280  1324 raft_consensus.cc:366] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.355952  1324 raft_consensus.cc:392] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.356216  1324 raft_consensus.cc:787] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.357136  1324 consensus_queue.cc:261] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.336685  1181 ts_tablet_manager.cc:1480] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.045s	user 0.007s	sys 0.000s
I20261120 03:00:54.359941  1324 ts_tablet_manager.cc:1480] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.007s	user 0.005s	sys 0.001s
I20261120 03:00:54.360877  1324 tablet_bootstrap.cc:492] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:54.364306  1461 raft_consensus.cc:534] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:54.364504  1181 tablet_bootstrap.cc:492] T 00dd2ffad1124244b171b2657d38a708 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:54.364760  1461 raft_consensus.cc:556] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.366729  1461 leader_election.cc:290] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:54.368644   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "daae644853734cd5818a46ecaa5ce7df" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:54.369400   920 raft_consensus.cc:2062] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:54.370707  1391 raft_consensus.cc:534] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:54.370956   710 leader_election.cc:304] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:54.371134  1391 raft_consensus.cc:556] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.371798  1461 raft_consensus.cc:3024] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:54.372114  1461 raft_consensus.cc:534] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:54.372407  1461 raft_consensus.cc:3296] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:54.373020  1391 leader_election.cc:290] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:54.375881   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5a3fe1499d0b4fab9f9949c8ded0a8dc" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:54.376477  1181 tablet_bootstrap.cc:654] T 00dd2ffad1124244b171b2657d38a708 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.378319  1461 raft_consensus.cc:556] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.384922  1072 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "daae644853734cd5818a46ecaa5ce7df" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:54.395977   709 leader_election.cc:343] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:54.397794  1324 tablet_bootstrap.cc:654] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.412489  1181 tablet_bootstrap.cc:492] T 00dd2ffad1124244b171b2657d38a708 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:54.412933  1181 ts_tablet_manager.cc:1449] T 00dd2ffad1124244b171b2657d38a708 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.049s	user 0.010s	sys 0.003s
I20261120 03:00:54.420603  1181 raft_consensus.cc:366] T 00dd2ffad1124244b171b2657d38a708 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.422983  1181 raft_consensus.cc:392] T 00dd2ffad1124244b171b2657d38a708 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.423251  1181 raft_consensus.cc:787] T 00dd2ffad1124244b171b2657d38a708 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.301668  1180 ts_tablet_manager.cc:1480] T fbbe19dadb08455285078e239d81bcca P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.045s	user 0.005s	sys 0.000s
I20261120 03:00:54.324898   902 tablet_service.cc:1542] Processing DeleteTablet for tablet a08aa5ddcf1a4061a70275a8eecb1856 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:53 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:54.428972  1181 consensus_queue.cc:261] T 00dd2ffad1124244b171b2657d38a708 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.379387   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5a3fe1499d0b4fab9f9949c8ded0a8dc" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:54.432988  1181 ts_tablet_manager.cc:1480] T 00dd2ffad1124244b171b2657d38a708 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.020s	user 0.006s	sys 0.001s
I20261120 03:00:54.435904  1324 tablet_bootstrap.cc:492] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:54.436301  1324 ts_tablet_manager.cc:1449] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.076s	user 0.012s	sys 0.010s
I20261120 03:00:54.438531  1324 raft_consensus.cc:366] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.439249  1324 raft_consensus.cc:392] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.439494  1324 raft_consensus.cc:787] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.440080  1181 tablet_bootstrap.cc:492] T d27099bc667b4a1d880af7cec2715bb8 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:54.440128  1324 consensus_queue.cc:261] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.443823  1461 leader_election.cc:290] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:54.445990  1324 ts_tablet_manager.cc:1480] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.009s	user 0.005s	sys 0.001s
I20261120 03:00:54.446835  1392 ts_tablet_manager.cc:1975] T fa57d0c282964e37b9133ea57f458a51 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:54.447228  1392 log.cc:1199] T fa57d0c282964e37b9133ea57f458a51 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/fa57d0c282964e37b9133ea57f458a51
I20261120 03:00:54.447886  1392 ts_tablet_manager.cc:1996] T fa57d0c282964e37b9133ea57f458a51 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:54.447808  1072 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "daae644853734cd5818a46ecaa5ce7df" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:54.448253  1324 tablet_bootstrap.cc:492] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
W20261120 03:00:54.450659   709 leader_election.cc:343] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:54.453712  1324 tablet_bootstrap.cc:654] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.458037   502 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_a121a467a36d4d159b7b3e108e3a00ea"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:54.459681   502 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_a121a467a36d4d159b7b3e108e3a00ea in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:54.462302   493 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 88b9e9692a874ae993dde4b9e2586a70
I20261120 03:00:54.462294   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "daae644853734cd5818a46ecaa5ce7df" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:54.461526   496 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_a925e1ca8a3d4ee18ad6481a8e3c1a73" } modify_external_catalogs: true
I20261120 03:00:54.462915   919 raft_consensus.cc:3296] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:54.463867   919 raft_consensus.cc:2062] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:54.463901   496 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_a925e1ca8a3d4ee18ad6481a8e3c1a73" } modify_external_catalogs: true
I20261120 03:00:54.464298   493 catalog_manager.cc:6952] ReplaceTablet: tablet 88b9e9692a874ae993dde4b9e2586a70 deleted and replaced by tablet 9e42a416be884341994d2d5395286798
I20261120 03:00:54.463881  1180 tablet_bootstrap.cc:492] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:54.467911  1050 tablet_service.cc:1542] Processing DeleteTablet for tablet 88b9e9692a874ae993dde4b9e2586a70 with delete_type TABLET_DATA_DELETED (replaced by tablet 9e42a416be884341994d2d5395286798) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:54.468550   754 tablet_service.cc:1542] Processing DeleteTablet for tablet 88b9e9692a874ae993dde4b9e2586a70 with delete_type TABLET_DATA_DELETED (replaced by tablet 9e42a416be884341994d2d5395286798) from {username='slave'} at 127.0.0.1:52644
W20261120 03:00:54.470791   497 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet b6f0ff2246cd4772af47812b699e93e8 consensus state has no leader
I20261120 03:00:54.472679  1181 tablet_bootstrap.cc:654] T d27099bc667b4a1d880af7cec2715bb8 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.456009   495 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_ec13480d9a854133bbc0df3c11fa69a6"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:54.475070   495 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_ec13480d9a854133bbc0df3c11fa69a6 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:54.497449   902 tablet_service.cc:1542] Processing DeleteTablet for tablet 88b9e9692a874ae993dde4b9e2586a70 with delete_type TABLET_DATA_DELETED (replaced by tablet 9e42a416be884341994d2d5395286798) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:54.518361  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 58) took 1926 ms. Trace:
I20261120 03:00:54.518729  1392 rpcz_store.cc:276] 1120 03:00:52.592428 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:52.607231 (+ 14803us) service_pool.cc:224] Handling call
1120 03:00:54.306823 (+1699592us) ts_tablet_manager.cc:1197] Deleting tablet fa57d0c282964e37b9133ea57f458a51
1120 03:00:54.306868 (+    45us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:54.361706 (+ 54838us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:54.446700 (+ 84994us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:54.518290 (+ 71590us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1694526}
I20261120 03:00:54.520164  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 61) took 1448 ms. Trace:
I20261120 03:00:54.520488  1392 rpcz_store.cc:276] 1120 03:00:53.076305 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.076489 (+   184us) service_pool.cc:224] Handling call
1120 03:00:54.519809 (+1443320us) ts_tablet_manager.cc:1197] Deleting tablet 200b234da897419a82bf5844620adf24
1120 03:00:54.519853 (+    44us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:54.520110 (+   257us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1441068}
I20261120 03:00:54.521613  1392 tablet_replica.cc:333] T 00dd2ffad1124244b171b2657d38a708 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:54.522886   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet fa57d0c282964e37b9133ea57f458a51 (table default.Table_dd2f7617ff0e401b9b862d5991110dd5 [id=3955f41b6d4d41bf8ba2c81e4ac7f4a8]) successfully deleted
W20261120 03:00:54.523540   486 catalog_manager.cc:5061] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 200b234da897419a82bf5844620adf24 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 200b234da897419a82bf5844620adf24 already in progress: creating tablet
I20261120 03:00:54.523922  1012 leader_election.cc:304] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:54.524447  1392 raft_consensus.cc:2440] T 00dd2ffad1124244b171b2657d38a708 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:54.525054  1392 raft_consensus.cc:2469] T 00dd2ffad1124244b171b2657d38a708 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:54.526340   496 catalog_manager.cc:6146] Not sending DeleteTablet requests; no consensus state for tablet bf5a19fc3b434a508e3402e3b1648654
I20261120 03:00:54.526309  1290 raft_consensus.cc:2966] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:54.527419  1392 ts_tablet_manager.cc:1962] T 00dd2ffad1124244b171b2657d38a708 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:54.531600   496 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_90a09b39479a4615b0dccfc155f2b7e3"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:54.533295   496 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_90a09b39479a4615b0dccfc155f2b7e3 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:54.535534  1180 tablet_bootstrap.cc:654] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.536649   502 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_61cb1e03b7e442a8822e32a2295866f3" } modify_external_catalogs: true
I20261120 03:00:54.537212   502 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_61cb1e03b7e442a8822e32a2295866f3" } modify_external_catalogs: true
I20261120 03:00:54.543766  1479 raft_consensus.cc:534] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:54.544354  1479 raft_consensus.cc:556] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:54.549008   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "33e6cd23a3474034953c9b1a37b79acd" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:54.550836   710 leader_election.cc:304] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:54.551533  1405 raft_consensus.cc:534] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:54.552093  1461 raft_consensus.cc:3024] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:54.551952  1405 raft_consensus.cc:556] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.552589  1461 raft_consensus.cc:744] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:54.557809  1072 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "33e6cd23a3474034953c9b1a37b79acd" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:54.558949  1405 leader_election.cc:290] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:54.558566  1461 consensus_queue.cc:238] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.566038  1181 tablet_bootstrap.cc:492] T d27099bc667b4a1d880af7cec2715bb8 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
W20261120 03:00:54.566708   710 leader_election.cc:343] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:54.567346  1181 ts_tablet_manager.cc:1449] T d27099bc667b4a1d880af7cec2715bb8 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.134s	user 0.013s	sys 0.006s
I20261120 03:00:54.569808  1181 raft_consensus.cc:366] T d27099bc667b4a1d880af7cec2715bb8 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:54.570264   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "fbbe19dadb08455285078e239d81bcca" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:54.570533  1181 raft_consensus.cc:392] T d27099bc667b4a1d880af7cec2715bb8 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.570788  1181 raft_consensus.cc:787] T d27099bc667b4a1d880af7cec2715bb8 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.570958   919 raft_consensus.cc:2062] T fbbe19dadb08455285078e239d81bcca P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:54.571480  1181 consensus_queue.cc:261] T d27099bc667b4a1d880af7cec2715bb8 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:54.573071  1072 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "fbbe19dadb08455285078e239d81bcca" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:54.574474   709 leader_election.cc:343] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:54.574803  1180 tablet_bootstrap.cc:492] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:54.574856   709 leader_election.cc:304] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:54.575198  1180 ts_tablet_manager.cc:1449] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.145s	user 0.008s	sys 0.002s
I20261120 03:00:54.576054  1289 raft_consensus.cc:534] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:54.576465  1289 raft_consensus.cc:556] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:54.576894   710 leader_election.cc:304] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:54.577689  1180 raft_consensus.cc:366] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.578321  1180 raft_consensus.cc:392] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.578406  1289 leader_election.cc:290] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:54.578565  1180 raft_consensus.cc:787] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.575806  1479 leader_election.cc:290] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:54.579170  1180 consensus_queue.cc:261] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:54.580453   709 leader_election.cc:343] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:54.581888  1072 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "68179488b9bc4e03b65a9f1d434c95b5" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:54.583192  1479 raft_consensus.cc:2966] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:54.583839  1405 raft_consensus.cc:3024] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:54.584225  1405 raft_consensus.cc:534] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
W20261120 03:00:54.584300   709 leader_election.cc:343] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:54.584549  1405 raft_consensus.cc:3296] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:54.585291  1405 raft_consensus.cc:556] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.589021   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "68179488b9bc4e03b65a9f1d434c95b5" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:00:54.590373   710 leader_election.cc:343] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:54.590814   710 leader_election.cc:304] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:54.595034  1180 ts_tablet_manager.cc:1480] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.019s	user 0.005s	sys 0.000s
I20261120 03:00:54.603129  1180 tablet_bootstrap.cc:492] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:54.602260  1479 raft_consensus.cc:2966] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:54.608268  1181 ts_tablet_manager.cc:1480] T d27099bc667b4a1d880af7cec2715bb8 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.041s	user 0.006s	sys 0.000s
I20261120 03:00:54.611922  1324 tablet_bootstrap.cc:492] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:54.612363  1324 ts_tablet_manager.cc:1449] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.164s	user 0.024s	sys 0.015s
I20261120 03:00:54.614701  1324 raft_consensus.cc:366] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.615334  1324 raft_consensus.cc:392] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.615577  1324 raft_consensus.cc:787] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.615628   495 master_service.cc:965] ReplaceTablet: received request to replace tablet 68179488b9bc4e03b65a9f1d434c95b5 from {username='slave'} at 127.0.0.1:45640
I20261120 03:00:54.616272  1324 consensus_queue.cc:261] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.618512  1479 consensus_queue.cc:1049] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:54.620023  1324 ts_tablet_manager.cc:1480] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.007s	user 0.006s	sys 0.001s
I20261120 03:00:54.620891  1324 tablet_bootstrap.cc:492] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:54.625412  1181 tablet_bootstrap.cc:492] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:54.628494   497 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_ab53c180c3b447daad84fe3018f841cc"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:54.629541  1180 tablet_bootstrap.cc:654] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:54.630298   497 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_ab53c180c3b447daad84fe3018f841cc in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:54.637234  1324 tablet_bootstrap.cc:654] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.644667  1295 ts_tablet_manager.cc:1975] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:54.645046  1295 log.cc:1199] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/fa57d0c282964e37b9133ea57f458a51
I20261120 03:00:54.645871  1295 ts_tablet_manager.cc:1996] T fa57d0c282964e37b9133ea57f458a51 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:54.646736  1185 ts_tablet_manager.cc:1975] T ce612f9a121b4b18908724d2726e156f P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:54.647132  1185 log.cc:1199] T ce612f9a121b4b18908724d2726e156f P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/ce612f9a121b4b18908724d2726e156f
I20261120 03:00:54.649070  1185 ts_tablet_manager.cc:1996] T ce612f9a121b4b18908724d2726e156f P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:54.649089  1180 tablet_bootstrap.cc:492] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:54.649472  1180 ts_tablet_manager.cc:1449] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.047s	user 0.003s	sys 0.007s
I20261120 03:00:54.657641  1072 raft_consensus.cc:3296] T 5fee8dff291c43e9be0fb10b308718ce P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:54.659914  1180 raft_consensus.cc:366] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:54.660530  1180 raft_consensus.cc:392] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.660784  1180 raft_consensus.cc:787] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.661649  1180 consensus_queue.cc:261] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:54.664882  1180 ts_tablet_manager.cc:1480] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.015s	user 0.003s	sys 0.004s
I20261120 03:00:54.666726  1181 tablet_bootstrap.cc:654] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.672497  1180 tablet_bootstrap.cc:492] T 33e6cd23a3474034953c9b1a37b79acd P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:54.675539   493 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_9498dfca4c8544c0a19cd826d27d441e" } new_table_name: "default.Table_faa63585fa0a40f7b9fe1e97441cdc7a" modify_external_catalogs: true
I20261120 03:00:54.678148  1180 tablet_bootstrap.cc:654] T 33e6cd23a3474034953c9b1a37b79acd P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:54.679162   487 catalog_manager.cc:4863] Async tablet task AlterTable RPC for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668], current schema version=2) failed: Not found: failed to reset TS proxy: No leader found: Tablet b6f0ff2246cd4772af47812b699e93e8 consensus state has no leader
I20261120 03:00:54.684593  1181 tablet_bootstrap.cc:492] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:54.685043  1181 ts_tablet_manager.cc:1449] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.060s	user 0.007s	sys 0.003s
I20261120 03:00:54.687479  1181 raft_consensus.cc:366] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.688100  1181 raft_consensus.cc:392] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.688345  1181 raft_consensus.cc:787] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.691991  1181 consensus_queue.cc:261] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.694787  1181 ts_tablet_manager.cc:1480] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.010s	user 0.005s	sys 0.001s
I20261120 03:00:54.696108  1180 tablet_bootstrap.cc:492] T 33e6cd23a3474034953c9b1a37b79acd P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:54.696483  1180 ts_tablet_manager.cc:1449] T 33e6cd23a3474034953c9b1a37b79acd P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.025s	user 0.009s	sys 0.001s
I20261120 03:00:54.698818  1181 tablet_bootstrap.cc:492] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:54.698832  1180 raft_consensus.cc:366] T 33e6cd23a3474034953c9b1a37b79acd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:54.699422  1180 raft_consensus.cc:392] T 33e6cd23a3474034953c9b1a37b79acd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.699664  1180 raft_consensus.cc:787] T 33e6cd23a3474034953c9b1a37b79acd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.707509  1180 consensus_queue.cc:261] T 33e6cd23a3474034953c9b1a37b79acd P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
W20261120 03:00:54.714424   493 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 0db7db25d779459ebde532f938493975 consensus state has no leader
I20261120 03:00:54.716122  1495 tablet.cc:1872] T 5fee8dff291c43e9be0fb10b308718ce P 823abdbd90ba47bbacad0f8dcff78234: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20261120 03:00:54.707000  1289 raft_consensus.cc:534] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:54.720467  1405 leader_election.cc:290] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:54.722409  1180 ts_tablet_manager.cc:1480] T 33e6cd23a3474034953c9b1a37b79acd P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.026s	user 0.005s	sys 0.001s
I20261120 03:00:54.725029  1180 tablet_bootstrap.cc:492] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:54.731683  1079 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "fbbe19dadb08455285078e239d81bcca" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:54.731967  1180 tablet_bootstrap.cc:654] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.732105   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "fbbe19dadb08455285078e239d81bcca" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:54.732658   920 raft_consensus.cc:3296] T fbbe19dadb08455285078e239d81bcca P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
W20261120 03:00:54.733466   709 leader_election.cc:343] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:54.733560   920 raft_consensus.cc:2062] T fbbe19dadb08455285078e239d81bcca P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:54.739529  1181 tablet_bootstrap.cc:654] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.746809  1392 ts_tablet_manager.cc:1975] T 00dd2ffad1124244b171b2657d38a708 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:54.747287  1392 log.cc:1199] T 00dd2ffad1124244b171b2657d38a708 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/00dd2ffad1124244b171b2657d38a708
I20261120 03:00:54.748093  1392 ts_tablet_manager.cc:1996] T 00dd2ffad1124244b171b2657d38a708 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:54.751151  1181 tablet_bootstrap.cc:492] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:54.751555  1181 ts_tablet_manager.cc:1449] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.056s	user 0.005s	sys 0.004s
I20261120 03:00:54.754001  1181 raft_consensus.cc:366] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:54.754614  1181 raft_consensus.cc:392] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.754858  1181 raft_consensus.cc:787] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.755426  1181 consensus_queue.cc:261] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:54.757836  1181 ts_tablet_manager.cc:1480] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.006s	user 0.006s	sys 0.000s
I20261120 03:00:54.758419   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet ce612f9a121b4b18908724d2726e156f (table default.Table_f601bffe6ddb49b3a3c589ac7b25235c [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]) successfully deleted
I20261120 03:00:54.758615  1181 tablet_bootstrap.cc:492] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
W20261120 03:00:54.762967   486 catalog_manager.cc:5061] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 88b9e9692a874ae993dde4b9e2586a70 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 88b9e9692a874ae993dde4b9e2586a70 already in progress: creating tablet
I20261120 03:00:54.749737  1289 raft_consensus.cc:556] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.768955  1072 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1885397e42774a89bb347bc776284bc0" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:54.771127  1181 tablet_bootstrap.cc:654] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:54.771341   709 leader_election.cc:343] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:54.776517  1289 leader_election.cc:290] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:54.781165  1181 tablet_bootstrap.cc:492] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:54.781554  1181 ts_tablet_manager.cc:1449] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.023s	user 0.009s	sys 0.000s
I20261120 03:00:54.785597  1181 raft_consensus.cc:366] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:54.786252  1181 raft_consensus.cc:392] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.789742  1181 raft_consensus.cc:787] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.791329  1181 consensus_queue.cc:261] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:54.798811  1181 ts_tablet_manager.cc:1480] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.017s	user 0.007s	sys 0.000s
W20261120 03:00:54.799605   709 consensus_peers.cc:597] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:54.801786  1181 tablet_bootstrap.cc:492] T a1f6197c60dd4dfb8aeb5c38373aab21 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:54.802772   493 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_8f246cc5b3c24316a8c07be0aff44a59"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:54.803260   501 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_43f3d907270e4093a84918b0a505ecc9"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:54.804615   493 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_8f246cc5b3c24316a8c07be0aff44a59 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:00:54.804827   501 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_43f3d907270e4093a84918b0a505ecc9 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:54.806828   496 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_cfa8989fb1134d908ab28bd6ebb80563" } modify_external_catalogs: true
I20261120 03:00:54.807308   496 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_cfa8989fb1134d908ab28bd6ebb80563" } modify_external_catalogs: true
I20261120 03:00:54.811731  1295 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 71) took 2303 ms. Trace:
I20261120 03:00:54.812088  1295 rpcz_store.cc:276] 1120 03:00:52.504375 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:52.505289 (+   914us) service_pool.cc:224] Handling call
1120 03:00:54.295281 (+1789992us) ts_tablet_manager.cc:1197] Deleting tablet fa57d0c282964e37b9133ea57f458a51
1120 03:00:54.295333 (+    52us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:54.546318 (+250985us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:54.644529 (+ 98211us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:54.808021 (+163492us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1787586}
I20261120 03:00:54.813024  1181 tablet_bootstrap.cc:654] T a1f6197c60dd4dfb8aeb5c38373aab21 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.814337   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1885397e42774a89bb347bc776284bc0" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:54.815865  1180 tablet_bootstrap.cc:492] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:54.816293  1180 ts_tablet_manager.cc:1449] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.092s	user 0.012s	sys 0.024s
I20261120 03:00:54.816507   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet fa57d0c282964e37b9133ea57f458a51 (table default.Table_dd2f7617ff0e401b9b862d5991110dd5 [id=3955f41b6d4d41bf8ba2c81e4ac7f4a8]) successfully deleted
I20261120 03:00:54.817449  1295 tablet_replica.cc:333] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:54.818166  1295 raft_consensus.cc:2440] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:54.820417  1180 raft_consensus.cc:366] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.821058  1180 raft_consensus.cc:392] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.821303  1180 raft_consensus.cc:787] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.822209  1295 pending_rounds.cc:70] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440: Trying to abort 1 pending ops.
I20261120 03:00:54.822438  1295 pending_rounds.cc:77] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7352908402572042240 op_type: NO_OP noop_request { }
I20261120 03:00:54.826845  1295 raft_consensus.cc:3115] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: NO_OP replication failed: Aborted: Op aborted
I20261120 03:00:54.828035  1295 raft_consensus.cc:2469] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:54.830087  1295 ts_tablet_manager.cc:1962] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:54.831985  1504 raft_consensus.cc:534] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:54.833856  1504 raft_consensus.cc:556] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:54.836031   710 leader_election.cc:343] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:54.836463   710 leader_election.cc:304] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:54.847663   497 master_service.cc:965] ReplaceTablet: received request to replace tablet a6ae8007455d4be3b50672b9c6b0fd8f from {username='slave'} at 127.0.0.1:45640
I20261120 03:00:54.849529  1391 raft_consensus.cc:534] T d27099bc667b4a1d880af7cec2715bb8 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:54.849990  1391 raft_consensus.cc:556] T d27099bc667b4a1d880af7cec2715bb8 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:54.851405  1181 tablet_bootstrap.cc:492] T a1f6197c60dd4dfb8aeb5c38373aab21 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:54.851805  1181 ts_tablet_manager.cc:1449] T a1f6197c60dd4dfb8aeb5c38373aab21 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.050s	user 0.014s	sys 0.001s
I20261120 03:00:54.852487  1391 leader_election.cc:290] T d27099bc667b4a1d880af7cec2715bb8 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:54.852772  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 62) took 1756 ms. Trace:
I20261120 03:00:54.853137  1392 rpcz_store.cc:276] 1120 03:00:53.097389 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.097818 (+   429us) service_pool.cc:224] Handling call
1120 03:00:54.521469 (+1423651us) ts_tablet_manager.cc:1197] Deleting tablet 00dd2ffad1124244b171b2657d38a708
1120 03:00:54.521508 (+    39us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:54.640473 (+118965us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:54.746637 (+106164us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:54.852697 (+106060us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1423113}
I20261120 03:00:54.854171  1181 raft_consensus.cc:366] T a1f6197c60dd4dfb8aeb5c38373aab21 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.854609  1289 raft_consensus.cc:2966] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:54.854809  1181 raft_consensus.cc:392] T a1f6197c60dd4dfb8aeb5c38373aab21 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.854805   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 00dd2ffad1124244b171b2657d38a708 (table default.Table_e07b2ba4d9864d148d7088622b10e03c [id=201bf626c7d643d7a98a680b1e714ba3]) successfully deleted
I20261120 03:00:54.855057  1181 raft_consensus.cc:787] T a1f6197c60dd4dfb8aeb5c38373aab21 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.856447  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 66) took 1370 ms. Trace:
I20261120 03:00:54.856681   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d27099bc667b4a1d880af7cec2715bb8" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:54.856750  1392 rpcz_store.cc:276] 1120 03:00:53.485714 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.485849 (+   135us) service_pool.cc:224] Handling call
1120 03:00:54.856064 (+1370215us) ts_tablet_manager.cc:1197] Deleting tablet 1885397e42774a89bb347bc776284bc0
1120 03:00:54.856106 (+    42us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:54.856393 (+   287us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1369445}
W20261120 03:00:54.858012   486 catalog_manager.cc:5061] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 1885397e42774a89bb347bc776284bc0 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 1885397e42774a89bb347bc776284bc0 already in progress: creating tablet
I20261120 03:00:54.860164  1181 consensus_queue.cc:261] T a1f6197c60dd4dfb8aeb5c38373aab21 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.863055  1181 ts_tablet_manager.cc:1480] T a1f6197c60dd4dfb8aeb5c38373aab21 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.011s	user 0.006s	sys 0.001s
I20261120 03:00:54.864022  1181 tablet_bootstrap.cc:492] T fbbe19dadb08455285078e239d81bcca P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:54.870232  1181 tablet_bootstrap.cc:654] T fbbe19dadb08455285078e239d81bcca P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.871202  1289 raft_consensus.cc:534] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:54.871383  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 70) took 1237 ms. Trace:
I20261120 03:00:54.871734  1392 rpcz_store.cc:276] 1120 03:00:53.622093 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.622224 (+   131us) service_pool.cc:224] Handling call
1120 03:00:54.858940 (+1236716us) ts_tablet_manager.cc:1197] Deleting tablet 020be7f7cda0467aab56f657cbe8d0ad
1120 03:00:54.858978 (+    38us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:54.859203 (+   225us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1236152}
I20261120 03:00:54.871784  1289 raft_consensus.cc:556] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:54.873162   486 catalog_manager.cc:5061] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 020be7f7cda0467aab56f657cbe8d0ad because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 020be7f7cda0467aab56f657cbe8d0ad already in progress: creating tablet
I20261120 03:00:54.874358  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 74) took 1047 ms. Trace:
I20261120 03:00:54.874955  1289 leader_election.cc:290] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:54.874711  1392 rpcz_store.cc:276] 1120 03:00:53.826562 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.826744 (+   182us) service_pool.cc:224] Handling call
1120 03:00:54.874014 (+1047270us) ts_tablet_manager.cc:1197] Deleting tablet 9f73b223756845bb8af7b7fd96c0f9fc
1120 03:00:54.874065 (+    51us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:54.874290 (+   225us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1046607}
W20261120 03:00:54.876570   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 9f73b223756845bb8af7b7fd96c0f9fc because the tablet was not found. No further retry: Not found: Tablet not found: 9f73b223756845bb8af7b7fd96c0f9fc
I20261120 03:00:54.877672  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 76) took 1048 ms. Trace:
I20261120 03:00:54.877849   710 leader_election.cc:304] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:54.878006  1392 rpcz_store.cc:276] 1120 03:00:53.828990 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.829135 (+   145us) service_pool.cc:224] Handling call
1120 03:00:54.877380 (+1048245us) ts_tablet_manager.cc:1197] Deleting tablet 1cf6d449fa2845a7b1f4cbd1de639d20
1120 03:00:54.877426 (+    46us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:54.877597 (+   171us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1047506}
I20261120 03:00:54.878691  1289 raft_consensus.cc:3024] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:54.879138  1289 raft_consensus.cc:744] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
W20261120 03:00:54.879477   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 1cf6d449fa2845a7b1f4cbd1de639d20 because the tablet was not found. No further retry: Not found: Tablet not found: 1cf6d449fa2845a7b1f4cbd1de639d20
I20261120 03:00:54.879971  1289 consensus_queue.cc:238] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.880683  1072 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "020be7f7cda0467aab56f657cbe8d0ad" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:54.882238  1181 tablet_bootstrap.cc:492] T fbbe19dadb08455285078e239d81bcca P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
W20261120 03:00:54.882164   709 leader_election.cc:343] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:54.882625  1181 ts_tablet_manager.cc:1449] T fbbe19dadb08455285078e239d81bcca P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.019s	user 0.010s	sys 0.007s
I20261120 03:00:54.821918  1180 consensus_queue.cc:261] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.877022   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "020be7f7cda0467aab56f657cbe8d0ad" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:54.885519   920 raft_consensus.cc:2062] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:54.883399   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d27099bc667b4a1d880af7cec2715bb8" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:54.886495  1180 ts_tablet_manager.cc:1480] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.070s	user 0.007s	sys 0.001s
I20261120 03:00:54.887356  1180 tablet_bootstrap.cc:492] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:54.888602  1012 leader_election.cc:304] T d27099bc667b4a1d880af7cec2715bb8 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:54.889357  1391 raft_consensus.cc:2966] T d27099bc667b4a1d880af7cec2715bb8 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:54.892385  1392 tablet_replica.cc:333] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:54.896683  1072 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ce612f9a121b4b18908724d2726e156f" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:54.896955  1504 leader_election.cc:290] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:54.899410  1324 tablet_bootstrap.cc:492] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:54.899881  1324 ts_tablet_manager.cc:1449] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.279s	user 0.008s	sys 0.005s
I20261120 03:00:54.899803  1392 raft_consensus.cc:2440] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:54.897471  1180 tablet_bootstrap.cc:654] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.896610   501 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_c1e41dc463834422869aec7e3bace0f1"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:54.901507  1392 raft_consensus.cc:2469] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:54.904101  1181 raft_consensus.cc:366] T fbbe19dadb08455285078e239d81bcca P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.904735  1181 raft_consensus.cc:392] T fbbe19dadb08455285078e239d81bcca P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.904995  1181 raft_consensus.cc:787] T fbbe19dadb08455285078e239d81bcca P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.905452  1324 raft_consensus.cc:366] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.905757  1181 consensus_queue.cc:261] T fbbe19dadb08455285078e239d81bcca P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.907621  1381 raft_consensus.cc:534] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:54.908013  1381 raft_consensus.cc:556] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:54.908759  1181 ts_tablet_manager.cc:1480] T fbbe19dadb08455285078e239d81bcca P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.026s	user 0.005s	sys 0.000s
I20261120 03:00:54.909646  1181 tablet_bootstrap.cc:492] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:54.910768  1381 leader_election.cc:290] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:54.913481  1392 ts_tablet_manager.cc:1962] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:54.913929  1324 raft_consensus.cc:392] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.914347  1324 raft_consensus.cc:787] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.915186  1324 consensus_queue.cc:261] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.918339   493 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_ec13480d9a854133bbc0df3c11fa69a6" } new_table_name: "default.Table_cc56bf0ad8344f1a94d5da6e4d77e8c0" modify_external_catalogs: true
I20261120 03:00:54.918752  1181 tablet_bootstrap.cc:654] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.922621  1180 tablet_bootstrap.cc:492] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:54.923007  1180 ts_tablet_manager.cc:1449] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.036s	user 0.005s	sys 0.008s
I20261120 03:00:54.925431  1180 raft_consensus.cc:366] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.926010  1180 raft_consensus.cc:392] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.927551  1180 raft_consensus.cc:787] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.929251  1180 consensus_queue.cc:261] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.929651   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "33e6cd23a3474034953c9b1a37b79acd" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:54.931504  1324 ts_tablet_manager.cc:1480] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.031s	user 0.006s	sys 0.000s
I20261120 03:00:54.933799  1181 tablet_bootstrap.cc:492] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:54.934232  1181 ts_tablet_manager.cc:1449] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.025s	user 0.010s	sys 0.000s
I20261120 03:00:54.936533   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "33e6cd23a3474034953c9b1a37b79acd" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:54.936556  1324 tablet_bootstrap.cc:492] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:54.937240   777 raft_consensus.cc:2062] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:00:54.937049  1181 raft_consensus.cc:366] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:54.937639  1181 raft_consensus.cc:392] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.937881  1181 raft_consensus.cc:787] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.938462  1181 consensus_queue.cc:261] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
W20261120 03:00:54.939751   501 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_c1e41dc463834422869aec7e3bace0f1 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:54.940409  1181 ts_tablet_manager.cc:1480] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.006s	user 0.006s	sys 0.000s
I20261120 03:00:54.941558  1181 tablet_bootstrap.cc:492] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:54.931198  1180 ts_tablet_manager.cc:1480] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.008s	user 0.005s	sys 0.001s
I20261120 03:00:54.942554   919 raft_consensus.cc:2062] T 33e6cd23a3474034953c9b1a37b79acd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:00:54.943082   710 leader_election.cc:304] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:54.948191  1181 tablet_bootstrap.cc:654] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.950970  1011 leader_election.cc:304] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:00:54.942900  1180 tablet_bootstrap.cc:492] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:54.953758  1324 tablet_bootstrap.cc:654] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:54.962558  1180 tablet_bootstrap.cc:654] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:54.933169   857 leader_election.cc:343] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:54.963188  1479 raft_consensus.cc:3024] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:54.963544  1479 raft_consensus.cc:534] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:54.963874  1479 raft_consensus.cc:3296] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:54.953548  1295 ts_tablet_manager.cc:1975] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:54.964871  1295 log.cc:1199] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/200b234da897419a82bf5844620adf24
I20261120 03:00:54.964669  1479 raft_consensus.cc:556] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.965565  1295 ts_tablet_manager.cc:1996] T 200b234da897419a82bf5844620adf24 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
W20261120 03:00:54.967833   857 leader_election.cc:343] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: ce612f9a121b4b18908724d2726e156f
I20261120 03:00:54.968196   857 leader_election.cc:304] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:54.975660  1504 raft_consensus.cc:2966] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:54.977001  1324 tablet_bootstrap.cc:492] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:54.977208  1180 tablet_bootstrap.cc:492] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:54.977424  1324 ts_tablet_manager.cc:1449] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.042s	user 0.009s	sys 0.000s
I20261120 03:00:54.977593  1180 ts_tablet_manager.cc:1449] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.035s	user 0.011s	sys 0.000s
I20261120 03:00:54.979784  1324 raft_consensus.cc:366] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.979918  1180 raft_consensus.cc:366] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.980505  1180 raft_consensus.cc:392] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.980505  1324 raft_consensus.cc:392] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:54.980748  1180 raft_consensus.cc:787] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.980765  1324 raft_consensus.cc:787] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.981410  1324 consensus_queue.cc:261] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.981493  1180 consensus_queue.cc:261] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.984220  1180 ts_tablet_manager.cc:1480] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.006s	user 0.004s	sys 0.003s
I20261120 03:00:54.984771  1504 raft_consensus.cc:534] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:54.985347  1504 raft_consensus.cc:556] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:54.987023  1324 ts_tablet_manager.cc:1480] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.009s	user 0.007s	sys 0.000s
I20261120 03:00:54.988395  1504 leader_election.cc:290] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:54.991838  1324 tablet_bootstrap.cc:492] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:54.999023  1496 raft_consensus.cc:534] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:54.999517  1496 raft_consensus.cc:556] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.000232   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1885397e42774a89bb347bc776284bc0" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:55.000916   776 raft_consensus.cc:2062] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:55.001585  1324 tablet_bootstrap.cc:654] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:55.002760  1485 raft_consensus.cc:534] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:55.003198  1485 raft_consensus.cc:556] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.004511   857 leader_election.cc:304] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:55.005306  1504 raft_consensus.cc:3024] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:55.005630  1504 raft_consensus.cc:534] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:55.005910  1504 raft_consensus.cc:3296] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.006683  1504 raft_consensus.cc:556] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:55.008553  1011 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 57678.8us
I20261120 03:00:55.008635  1381 raft_consensus.cc:3024] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:55.009001  1381 raft_consensus.cc:534] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:55.009315  1381 raft_consensus.cc:3296] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.010030  1381 raft_consensus.cc:556] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:55.021224  1071 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0db7db25d779459ebde532f938493975" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:55.022498  1069 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1885397e42774a89bb347bc776284bc0" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:55.023864   857 leader_election.cc:343] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:55.008169  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "35f0ff39b36148e8abfdea3435a3b8b4" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:55.025138  1012 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 64173.2us
W20261120 03:00:55.026461   709 leader_election.cc:343] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:55.027163   709 leader_election.cc:343] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:54.954906  1072 raft_consensus.cc:3296] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.001502  1496 leader_election.cc:290] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:54.970878  1181 tablet_bootstrap.cc:492] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:55.030401  1181 ts_tablet_manager.cc:1449] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.089s	user 0.013s	sys 0.002s
I20261120 03:00:55.032737  1181 raft_consensus.cc:366] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.033326  1181 raft_consensus.cc:392] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.033563  1181 raft_consensus.cc:787] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:54.991144  1079 raft_consensus.cc:3296] T fbbe19dadb08455285078e239d81bcca P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.033806  1485 leader_election.cc:290] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:55.034132  1181 consensus_queue.cc:261] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.034979  1324 tablet_bootstrap.cc:492] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:55.035364  1324 ts_tablet_manager.cc:1449] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.044s	user 0.009s	sys 0.000s
I20261120 03:00:55.037631  1324 raft_consensus.cc:366] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.038367  1324 raft_consensus.cc:392] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.038607  1324 raft_consensus.cc:787] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:55.039265  1324 consensus_queue.cc:261] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.042533  1180 tablet_bootstrap.cc:492] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:55.043013  1324 ts_tablet_manager.cc:1480] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.007s	user 0.006s	sys 0.000s
I20261120 03:00:55.043793  1324 tablet_bootstrap.cc:492] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:55.049290  1180 tablet_bootstrap.cc:654] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:55.053112  1324 tablet_bootstrap.cc:654] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:55.058373  1295 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 73) took 2008 ms. Trace:
I20261120 03:00:55.058722  1295 rpcz_store.cc:276] 1120 03:00:53.049625 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.049749 (+   124us) service_pool.cc:224] Handling call
1120 03:00:54.813896 (+1764147us) ts_tablet_manager.cc:1197] Deleting tablet 200b234da897419a82bf5844620adf24
1120 03:00:54.817342 (+  3446us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:54.844621 (+ 27279us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:54.953415 (+108794us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:55.058297 (+104882us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1763539}
I20261120 03:00:55.059917  1295 tablet_replica.cc:333] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:55.060693  1295 raft_consensus.cc:2440] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:55.061246  1295 raft_consensus.cc:2469] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:55.062816   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 200b234da897419a82bf5844620adf24 (table default.Table_5303db2882934be1bfc2251ce51f2c8d [id=ae52d3d892e24bc38a8ff7d6db755c2d]) successfully deleted
I20261120 03:00:55.065021  1479 leader_election.cc:290] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:55.067270  1181 ts_tablet_manager.cc:1480] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.037s	user 0.009s	sys 0.001s
I20261120 03:00:55.068343  1295 ts_tablet_manager.cc:1962] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:55.069293   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "35f0ff39b36148e8abfdea3435a3b8b4" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:55.071587  1181 tablet_bootstrap.cc:492] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:55.072337  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "020be7f7cda0467aab56f657cbe8d0ad" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:55.072995  1078 raft_consensus.cc:3296] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.073364  1479 consensus_queue.cc:1049] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:55.076752  1504 leader_election.cc:290] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
W20261120 03:00:55.077749   710 leader_election.cc:343] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:55.078130   710 leader_election.cc:304] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:55.081032  1078 raft_consensus.cc:2062] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:55.081902  1069 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1885397e42774a89bb347bc776284bc0" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:55.082736  1181 tablet_bootstrap.cc:654] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:55.083328   857 leader_election.cc:343] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:55.086346   486 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.124:38133, user_credentials={real_user=slave}} blocked reactor thread for 122161us
I20261120 03:00:55.086874   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "33e6cd23a3474034953c9b1a37b79acd" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:55.087442   776 raft_consensus.cc:3296] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
W20261120 03:00:55.092025   493 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 0179a59749a14168a840e9d4e610be1f has no consensus state
I20261120 03:00:55.094241  1381 leader_election.cc:290] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Requested vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:55.093642   493 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_c1e302c0d12d4655b7871a1792728e02"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:55.096398   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "33e6cd23a3474034953c9b1a37b79acd" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:55.097434   924 raft_consensus.cc:3296] T 33e6cd23a3474034953c9b1a37b79acd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
W20261120 03:00:55.097684   493 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_c1e302c0d12d4655b7871a1792728e02 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:55.098415   924 raft_consensus.cc:2062] T 33e6cd23a3474034953c9b1a37b79acd P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
I20261120 03:00:55.099478  1479 raft_consensus.cc:2966] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:55.100929   776 raft_consensus.cc:2062] T 33e6cd23a3474034953c9b1a37b79acd P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
I20261120 03:00:55.107829  1461 raft_consensus.cc:534] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:55.108361  1461 raft_consensus.cc:556] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.110637   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "020be7f7cda0467aab56f657cbe8d0ad" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:55.111198   920 raft_consensus.cc:3296] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.111124   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1885397e42774a89bb347bc776284bc0" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:55.111673   777 raft_consensus.cc:3296] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.112192   920 raft_consensus.cc:2062] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:55.112996   777 raft_consensus.cc:2062] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:00:55.118809   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0db7db25d779459ebde532f938493975" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:00:55.123090   710 leader_election.cc:343] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:55.123476   710 leader_election.cc:304] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:55.124356  1461 leader_election.cc:290] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:55.125947  1180 tablet_bootstrap.cc:492] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:55.126422  1180 ts_tablet_manager.cc:1449] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.084s	user 0.015s	sys 0.007s
I20261120 03:00:55.131228  1180 raft_consensus.cc:366] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.131887  1180 raft_consensus.cc:392] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.132148  1180 raft_consensus.cc:787] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:55.132762  1496 raft_consensus.cc:2966] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:55.134377  1180 consensus_queue.cc:261] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.134951   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "88b9e9692a874ae993dde4b9e2586a70" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:55.137566  1180 ts_tablet_manager.cc:1480] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.011s	user 0.005s	sys 0.000s
I20261120 03:00:55.138382  1180 tablet_bootstrap.cc:492] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:55.139780  1069 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "88b9e9692a874ae993dde4b9e2586a70" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:55.141737   709 leader_election.cc:343] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:55.151907  1180 tablet_bootstrap.cc:654] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:55.152848   710 leader_election.cc:343] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:55.159852  1012 leader_election.cc:304] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:00:55.161532  1515 raft_consensus.cc:3024] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:55.162010  1515 raft_consensus.cc:744] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Becoming Leader. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Running, Role: LEADER
I20261120 03:00:55.162770  1515 consensus_queue.cc:238] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:55.171885  1181 tablet_bootstrap.cc:492] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:55.172266  1181 ts_tablet_manager.cc:1449] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.101s	user 0.010s	sys 0.002s
I20261120 03:00:55.177325   710 leader_election.cc:304] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
W20261120 03:00:55.177966   486 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.126:44467, user_credentials={real_user=slave}} blocked reactor thread for 89661us
I20261120 03:00:55.182660  1181 raft_consensus.cc:366] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.183320  1181 raft_consensus.cc:392] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.183579  1181 raft_consensus.cc:787] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:55.184378  1181 consensus_queue.cc:261] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.195654  1479 raft_consensus.cc:2966] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:55.195782  1391 consensus_queue.cc:1049] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:55.199968  1181 ts_tablet_manager.cc:1480] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.027s	user 0.006s	sys 0.001s
I20261120 03:00:55.200474  1515 consensus_queue.cc:1049] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:55.200863  1181 tablet_bootstrap.cc:492] T daae644853734cd5818a46ecaa5ce7df P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:55.202207  1180 tablet_bootstrap.cc:492] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:55.202574  1180 ts_tablet_manager.cc:1449] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.064s	user 0.010s	sys 0.013s
I20261120 03:00:55.205279  1180 raft_consensus.cc:366] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.205884  1180 raft_consensus.cc:392] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.206151  1180 raft_consensus.cc:787] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:55.206833  1180 consensus_queue.cc:261] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.209676  1180 ts_tablet_manager.cc:1480] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.007s	user 0.006s	sys 0.000s
I20261120 03:00:55.210093  1515 raft_consensus.cc:534] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:55.210525  1515 raft_consensus.cc:556] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.214826  1504 raft_consensus.cc:534] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:55.215286  1504 raft_consensus.cc:556] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.217374  1504 leader_election.cc:290] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:55.218531  1180 tablet_bootstrap.cc:492] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
W20261120 03:00:55.220813  1012 leader_election.cc:343] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Not found: Tablet not found: 136056e5242a42ef8fa4b40c989b5423
I20261120 03:00:55.224113  1180 tablet_bootstrap.cc:654] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:55.232821  1381 raft_consensus.cc:534] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:55.233299  1381 raft_consensus.cc:556] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:55.233801  1181 tablet_bootstrap.cc:654] T daae644853734cd5818a46ecaa5ce7df P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:55.235720  1381 leader_election.cc:290] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:55.240674  1180 tablet_bootstrap.cc:492] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:55.241065  1180 ts_tablet_manager.cc:1449] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.023s	user 0.012s	sys 0.000s
I20261120 03:00:55.242338  1504 raft_consensus.cc:534] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:55.242878  1504 raft_consensus.cc:556] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:55.243392  1180 raft_consensus.cc:366] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.244025  1180 raft_consensus.cc:392] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.244292  1180 raft_consensus.cc:787] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:55.246454  1504 leader_election.cc:290] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
W20261120 03:00:55.248497  1012 leader_election.cc:343] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Not found: Tablet not found: 200b234da897419a82bf5844620adf24
I20261120 03:00:55.250371  1180 consensus_queue.cc:261] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.251282  1391 raft_consensus.cc:534] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 3d30c2428730481db9b0ffd279dd3440)
I20261120 03:00:55.251820  1391 raft_consensus.cc:556] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.253422  1180 ts_tablet_manager.cc:1480] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.012s	user 0.005s	sys 0.001s
I20261120 03:00:55.254274  1180 tablet_bootstrap.cc:492] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:55.259424  1181 tablet_bootstrap.cc:492] T daae644853734cd5818a46ecaa5ce7df P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:55.259867  1181 ts_tablet_manager.cc:1449] T daae644853734cd5818a46ecaa5ce7df P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.059s	user 0.009s	sys 0.000s
I20261120 03:00:55.257929  1391 leader_election.cc:290] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:55.263224  1181 raft_consensus.cc:366] T daae644853734cd5818a46ecaa5ce7df P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.263831  1181 raft_consensus.cc:392] T daae644853734cd5818a46ecaa5ce7df P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.264101  1181 raft_consensus.cc:787] T daae644853734cd5818a46ecaa5ce7df P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
W20261120 03:00:55.264901  1012 leader_election.cc:343] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Not found: Tablet not found: 06c3bac919e342b09dd6af13aebc387b
I20261120 03:00:55.266198  1072 raft_consensus.cc:3296] T a1f6197c60dd4dfb8aeb5c38373aab21 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.267306  1180 tablet_bootstrap.cc:654] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:55.269618  1011 leader_election.cc:343] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: 136056e5242a42ef8fa4b40c989b5423
I20261120 03:00:55.270437  1011 leader_election.cc:304] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:55.264735  1181 consensus_queue.cc:261] T daae644853734cd5818a46ecaa5ce7df P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:55.276914   710 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 124115us
I20261120 03:00:55.277777  1181 ts_tablet_manager.cc:1480] T daae644853734cd5818a46ecaa5ce7df P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.018s	user 0.006s	sys 0.000s
I20261120 03:00:55.278760  1181 tablet_bootstrap.cc:492] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:55.282673  1180 tablet_bootstrap.cc:492] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:55.283053  1180 ts_tablet_manager.cc:1449] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.029s	user 0.008s	sys 0.004s
I20261120 03:00:55.285405  1180 raft_consensus.cc:366] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.286016  1180 raft_consensus.cc:392] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.286288  1180 raft_consensus.cc:787] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:55.288691  1515 leader_election.cc:290] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:55.294152  1290 raft_consensus.cc:2966] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20261120 03:00:55.295258  1011 leader_election.cc:343] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: 200b234da897419a82bf5844620adf24
I20261120 03:00:55.296698  1011 leader_election.cc:304] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:55.296821  1180 consensus_queue.cc:261] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.297338  1290 raft_consensus.cc:2966] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:55.298767  1180 ts_tablet_manager.cc:1480] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.015s	user 0.006s	sys 0.000s
I20261120 03:00:55.299610  1180 tablet_bootstrap.cc:492] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:55.306838  1069 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "168cb98bb1b242a5a0377bf292204d96" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:55.308593  1180 tablet_bootstrap.cc:654] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:55.309161  1069 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "68179488b9bc4e03b65a9f1d434c95b5" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:55.312166   857 leader_election.cc:343] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:55.322692   857 leader_election.cc:343] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:55.323964  1011 leader_election.cc:343] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: 06c3bac919e342b09dd6af13aebc387b
I20261120 03:00:55.323613   499 catalog_manager.cc:5856] T d27099bc667b4a1d880af7cec2715bb8 P 5beeefe8bef54caeac463d05d8c30fa6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65). New cstate: current_term: 1 leader_uuid: "5beeefe8bef54caeac463d05d8c30fa6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: HEALTHY } } }
I20261120 03:00:55.325023  1011 leader_election.cc:304] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:55.324689  1181 tablet_bootstrap.cc:654] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:55.328323  1290 raft_consensus.cc:2966] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20261120 03:00:55.354606   902 tablet_service.cc:1495] Processing CreateTablet for tablet dc4133a8f33b4902a6308f9bdd4f91d2 (DEFAULT_TABLE table=default.Table_0c88eb71972c46cd918b4abf3ee01e11 [id=e24a1e3bb5054f6198ebd21469ab6f3c]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:55.355940  1181 tablet_bootstrap.cc:492] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:55.356366  1181 ts_tablet_manager.cc:1449] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.078s	user 0.007s	sys 0.002s
I20261120 03:00:55.357928   499 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 9882c9cbfbd640a49e3e3c8904ffc1e1 (table default.Table_0879bd445c234e4faefff97cd088035d [id=53fb1849755c4debad2dbfbf37271fbd]). Expected version 1 got 0
I20261120 03:00:55.358829  1181 raft_consensus.cc:366] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.359489  1181 raft_consensus.cc:392] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.359740  1181 raft_consensus.cc:787] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:55.360390  1181 consensus_queue.cc:261] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:55.362929   494 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 5da3d2724dd34aed88e6b08983188004 has no consensus state
I20261120 03:00:55.363521   494 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:45640 (request call id 135) took 1540 ms. Trace:
I20261120 03:00:55.363833   494 rpcz_store.cc:276] 1120 03:00:53.822754 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.952581 (+129827us) service_pool.cc:224] Handling call
1120 03:00:54.069847 (+117266us) spinlock_profiling.cc:245] Waited 116 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07f626 00007f155ac007f4 00007f155ac00782 00007f155ac006b1 00007f155ac0062c 00007f155abff8e2 00007f1558e990bc 00007f1558e984f6
1120 03:00:54.070980 (+  1133us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:00:54.072266 (+  1286us) catalog_manager.cc:3769] Apply alter schema
1120 03:00:54.072979 (+   713us) catalog_manager.cc:3801] Acquired catalog manager lock
1120 03:00:54.073193 (+   214us) catalog_manager.cc:3972] Updating metadata on disk
1120 03:00:54.461639 (+388446us) catalog_manager.cc:4003] Committing alterations to in-memory state
1120 03:00:55.363457 (+901818us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:54.108884 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:54.109289 (+   405us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:54.109312 (+    23us) write_op.cc:440] Acquired schema lock
1120 03:00:54.109325 (+    13us) tablet.cc:710] Decoding operations
1120 03:00:54.109578 (+   253us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:54.109618 (+    40us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:54.109638 (+    20us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:00:54.109790 (+   152us) tablet.cc:758] Row locks acquired
1120 03:00:54.109802 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:00:54.109919 (+   117us) write_op.cc:270] Start()
1120 03:00:54.110001 (+    82us) write_op.cc:276] Timestamp: P: 1795143654109867 usec, L: 0
1120 03:00:54.110017 (+    16us) op_driver.cc:350] REPLICATION: starting
1120 03:00:54.110544 (+   527us) log.cc:844] Serialized 405 byte log entry
1120 03:00:54.455968 (+345424us) op_driver.cc:466] REPLICATION: finished
1120 03:00:54.456635 (+   667us) write_op.cc:301] APPLY: starting
1120 03:00:54.456694 (+    59us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:54.456788 (+    94us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:54.456892 (+   104us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:54.457036 (+   144us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:54.457182 (+   146us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:54.457221 (+    39us) write_op.cc:312] APPLY: finished
1120 03:00:54.457404 (+   183us) log.cc:844] Serialized 23 byte log entry
1120 03:00:54.457548 (+   144us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:54.457615 (+    67us) write_op.cc:459] Released schema lock
1120 03:00:54.457655 (+    40us) write_op.cc:346] FINISH: Updating metrics
1120 03:00:54.457874 (+   219us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:54.457898 (+    24us) write_op.cc:459] Released schema lock
Metrics: {"spinlock_wait_cycles":267713280,"child_traces":[["op",{"apply.queue_time_us":509,"apply.run_cpu_time_us":1202,"apply.run_wall_time_us":1201,"num_ops":1,"prepare.queue_time_us":25805,"prepare.run_cpu_time_us":1862,"prepare.run_wall_time_us":2803,"replication_time_us":345852}]]}
I20261120 03:00:55.372014  1181 ts_tablet_manager.cc:1480] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.015s	user 0.003s	sys 0.004s
I20261120 03:00:55.373020  1181 tablet_bootstrap.cc:492] T d736d65c3e0b419084b4934dad83f16a P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:55.378950   899 tablet_service.cc:1495] Processing CreateTablet for tablet 091dd73d27df4db09ab750747da7584e (DEFAULT_TABLE table=default.Table_3c9436f0bbcd41bb987053b52acc02cf [id=b349c04100df4df599ad63dd3802bdd8]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:55.388290   901 tablet_service.cc:1495] Processing CreateTablet for tablet 014bd9c83be34c31b90fe4bcf31f1649 (DEFAULT_TABLE table=default.Table_cfa8989fb1134d908ab28bd6ebb80563 [id=16bb3f7e35c54a8c936b4af5a66a7f17]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:55.388797  1522 raft_consensus.cc:534] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:55.391047   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "168cb98bb1b242a5a0377bf292204d96" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:55.390381  1522 raft_consensus.cc:556] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.397315   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "68179488b9bc4e03b65a9f1d434c95b5" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:55.398018   776 raft_consensus.cc:2062] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:55.403192  1290 raft_consensus.cc:534] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:55.403649  1290 raft_consensus.cc:556] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.404120  1181 tablet_bootstrap.cc:654] T d736d65c3e0b419084b4934dad83f16a P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:55.405855  1290 leader_election.cc:290] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:55.409576  1290 raft_consensus.cc:534] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
W20261120 03:00:55.411139  1011 leader_election.cc:343] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: ce612f9a121b4b18908724d2726e156f
I20261120 03:00:55.410059  1290 raft_consensus.cc:556] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.416482  1290 leader_election.cc:290] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:55.418391  1069 raft_consensus.cc:3296] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.420562  1181 tablet_bootstrap.cc:492] T d736d65c3e0b419084b4934dad83f16a P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:55.420974  1181 ts_tablet_manager.cc:1449] T d736d65c3e0b419084b4934dad83f16a P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.048s	user 0.009s	sys 0.000s
I20261120 03:00:55.423348  1181 raft_consensus.cc:366] T d736d65c3e0b419084b4934dad83f16a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.424024  1181 raft_consensus.cc:392] T d736d65c3e0b419084b4934dad83f16a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.424361  1181 raft_consensus.cc:787] T d736d65c3e0b419084b4934dad83f16a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:55.425105   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "4abb67f3819a45b9b05dde04842e2c9d" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:55.428687   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "4abb67f3819a45b9b05dde04842e2c9d" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:55.430550  1012 leader_election.cc:304] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:55.432881  1516 raft_consensus.cc:2966] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:55.433902  1181 consensus_queue.cc:261] T d736d65c3e0b419084b4934dad83f16a P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.437152   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ce612f9a121b4b18908724d2726e156f" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:55.437893   923 raft_consensus.cc:2062] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
W20261120 03:00:55.440357   857 leader_election.cc:343] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:55.440716   857 leader_election.cc:304] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:55.441592  1504 raft_consensus.cc:2966] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:55.449756   857 leader_election.cc:304] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:55.450963  1504 raft_consensus.cc:3024] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:55.451332  1504 raft_consensus.cc:534] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:55.451648  1504 raft_consensus.cc:3296] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.452489  1504 raft_consensus.cc:556] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:55.467087  1522 leader_election.cc:290] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:55.477682   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet dc4133a8f33b4902a6308f9bdd4f91d2. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:55.489068   901 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 014bd9c83be34c31b90fe4bcf31f1649. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:55.495250   899 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 091dd73d27df4db09ab750747da7584e. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:55.534688  1500 raft_consensus.cc:534] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:55.535161  1500 raft_consensus.cc:556] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.537240  1500 leader_election.cc:290] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:55.555227  1500 raft_consensus.cc:534] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:55.555680  1500 raft_consensus.cc:556] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.557528  1500 leader_election.cc:290] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:55.565156   857 leader_election.cc:304] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:55.566002  1500 raft_consensus.cc:3024] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:55.566447  1500 raft_consensus.cc:744] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:00:55.567181  1500 consensus_queue.cc:238] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.611363  1504 leader_election.cc:290] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:55.623947  1485 raft_consensus.cc:534] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:55.624435  1485 raft_consensus.cc:556] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.626313  1485 leader_election.cc:290] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:55.628185  1485 raft_consensus.cc:534] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:55.628669  1485 raft_consensus.cc:556] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.632282  1485 leader_election.cc:290] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:55.633638   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5c5c0ac869a1419fad9f61ec647027ee" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:55.635499   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b6f0ff2246cd4772af47812b699e93e8" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:55.636200   777 raft_consensus.cc:2062] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:55.637634   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "88b9e9692a874ae993dde4b9e2586a70" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:55.638305   777 raft_consensus.cc:2062] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:55.642350   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "68179488b9bc4e03b65a9f1d434c95b5" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:55.642935   777 raft_consensus.cc:3296] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.644002   777 raft_consensus.cc:2062] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:00:55.645955  1324 tablet_bootstrap.cc:492] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:55.646415  1324 ts_tablet_manager.cc:1449] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.603s	user 0.077s	sys 0.089s
I20261120 03:00:55.656498  1324 raft_consensus.cc:366] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.657217  1324 raft_consensus.cc:392] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.657447  1324 raft_consensus.cc:787] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:55.658130  1324 consensus_queue.cc:261] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.659873  1324 ts_tablet_manager.cc:1480] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.013s	user 0.004s	sys 0.000s
I20261120 03:00:55.660656  1324 tablet_bootstrap.cc:492] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:55.669138  1324 tablet_bootstrap.cc:654] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:55.681222   899 tablet_service.cc:1495] Processing CreateTablet for tablet b3f7103b59d443f9bc739083b8e023ce (DEFAULT_TABLE table=default.Table_555e4ecfadce486baefa8c3e2d1bc272 [id=def72d7bfc214aa5a942c18714c0173e]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:55.682421   899 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b3f7103b59d443f9bc739083b8e023ce. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:55.685463   901 tablet_service.cc:1495] Processing CreateTablet for tablet f376fdc03b3b4e3680171af6b8574c2e (DEFAULT_TABLE table=default.Table_7f80bbdf810749109360575194927d04 [id=382f9ae93f5e47baa2f361c7d853f5a9]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:55.685487  1012 leader_election.cc:304] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:55.686610   901 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f376fdc03b3b4e3680171af6b8574c2e. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:55.686450  1070 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "88b9e9692a874ae993dde4b9e2586a70" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:55.688776   857 leader_election.cc:343] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:55.692153  1071 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5c5c0ac869a1419fad9f61ec647027ee" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:55.693626   857 leader_election.cc:343] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:55.694147  1068 raft_consensus.cc:3296] T d27099bc667b4a1d880af7cec2715bb8 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.697654  1070 raft_consensus.cc:3296] T 5a3fe1499d0b4fab9f9949c8ded0a8dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.709600  1548 raft_consensus.cc:534] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 18331bef5c884dcab9dbd26bc2c82273)
I20261120 03:00:55.716410  1076 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d736d65c3e0b419084b4934dad83f16a" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:55.717609  1076 raft_consensus.cc:2062] T d736d65c3e0b419084b4934dad83f16a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:55.720033   709 leader_election.cc:304] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:00:55.721185  1496 raft_consensus.cc:3024] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:55.721520  1496 raft_consensus.cc:534] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:55.721825  1496 raft_consensus.cc:3296] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.722641  1496 raft_consensus.cc:556] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:55.723140   488 catalog_manager.cc:4863] Async tablet task AlterTable RPC for tablet 5da3d2724dd34aed88e6b08983188004 (table default.Table_d2bd3060da1545548f422436c9c7c22e [id=f5d860e17cd74d6aa7b08fbbb0f8c75a], current schema version=2) failed: Not found: failed to reset TS proxy: No leader found: Tablet 5da3d2724dd34aed88e6b08983188004 has no consensus state
I20261120 03:00:55.724668  1479 consensus_queue.cc:1049] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20261120 03:00:55.725239   857 consensus_peers.cc:597] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:55.733857  1548 raft_consensus.cc:556] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:55.739701  1324 tablet_bootstrap.cc:492] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:55.740119  1324 ts_tablet_manager.cc:1449] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.080s	user 0.016s	sys 0.015s
I20261120 03:00:55.742501  1324 raft_consensus.cc:366] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.742702   512 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e102e07459954b4ea13b65ab9fd5d18b" candidate_term: 4 candidate_status { last_received { term: 3 index: 173 } } ignore_live_leader: false dest_uuid: "18331bef5c884dcab9dbd26bc2c82273" is_pre_election: true
I20261120 03:00:55.743124  1324 raft_consensus.cc:392] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.743352  1324 raft_consensus.cc:787] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:55.746343  1548 leader_election.cc:290] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 18331bef5c884dcab9dbd26bc2c82273 (127.0.96.125:37411), 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133)
I20261120 03:00:55.750727  1324 consensus_queue.cc:261] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.752790  1324 ts_tablet_manager.cc:1480] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.012s	user 0.005s	sys 0.000s
I20261120 03:00:55.753702  1324 tablet_bootstrap.cc:492] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
W20261120 03:00:55.755951   411 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111) [suppressed 13 similar messages]
I20261120 03:00:55.759331  1324 tablet_bootstrap.cc:654] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:55.762442  1542 raft_consensus.cc:534] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:55.655365   710 leader_election.cc:304] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:55.762943  1542 raft_consensus.cc:556] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.763525  1554 raft_consensus.cc:3024] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:55.763957  1554 raft_consensus.cc:744] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:55.764705  1554 consensus_queue.cc:238] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.765296  1542 leader_election.cc:290] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
W20261120 03:00:55.581982   486 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.126:44467, user_credentials={real_user=slave}} blocked reactor thread for 276450us
I20261120 03:00:55.774619  1324 tablet_bootstrap.cc:492] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:55.775041  1324 ts_tablet_manager.cc:1449] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.022s	user 0.009s	sys 0.000s
I20261120 03:00:55.775445   754 tablet_service.cc:1495] Processing CreateTablet for tablet dc4133a8f33b4902a6308f9bdd4f91d2 (DEFAULT_TABLE table=default.Table_0c88eb71972c46cd918b4abf3ee01e11 [id=e24a1e3bb5054f6198ebd21469ab6f3c]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:55.776523   754 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet dc4133a8f33b4902a6308f9bdd4f91d2. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:55.777570  1324 raft_consensus.cc:366] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.778280  1324 raft_consensus.cc:392] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.778539  1324 raft_consensus.cc:787] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:55.779170  1324 consensus_queue.cc:261] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.791716  1324 ts_tablet_manager.cc:1480] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.016s	user 0.006s	sys 0.000s
I20261120 03:00:55.792641  1324 tablet_bootstrap.cc:492] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
W20261120 03:00:55.793069   710 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 137776us
I20261120 03:00:55.796181   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b6f0ff2246cd4772af47812b699e93e8" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:55.796947   920 raft_consensus.cc:2062] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:55.798310  1324 tablet_bootstrap.cc:654] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:55.801376   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d736d65c3e0b419084b4934dad83f16a" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:55.802045   920 raft_consensus.cc:2062] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:55.811322  1324 tablet_bootstrap.cc:492] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:55.811777  1324 ts_tablet_manager.cc:1449] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.019s	user 0.004s	sys 0.004s
I20261120 03:00:55.814260  1324 raft_consensus.cc:366] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.814891  1324 raft_consensus.cc:392] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.815133  1324 raft_consensus.cc:787] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:55.815732  1324 consensus_queue.cc:261] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:55.819536  1180 thread.cc:642] log-alloc [worker] (thread pool) Time spent creating pthread: real 0.509s	user 0.123s	sys 0.193s
W20261120 03:00:55.819922  1180 thread.cc:609] log-alloc [worker] (thread pool) Time spent starting thread: real 0.509s	user 0.123s	sys 0.193s
I20261120 03:00:55.826092  1180 tablet_bootstrap.cc:492] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:55.827337  1517 raft_consensus.cc:3024] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:55.827657  1324 ts_tablet_manager.cc:1480] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.016s	user 0.006s	sys 0.000s
I20261120 03:00:55.827543  1077 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "68179488b9bc4e03b65a9f1d434c95b5" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:55.827909  1517 raft_consensus.cc:534] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:55.828298  1517 raft_consensus.cc:3296] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.828555  1324 tablet_bootstrap.cc:492] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:55.828878  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b6f0ff2246cd4772af47812b699e93e8" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:55.829120  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b6f0ff2246cd4772af47812b699e93e8" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:55.826535  1180 ts_tablet_manager.cc:1449] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.527s	user 0.134s	sys 0.193s
I20261120 03:00:55.834064  1324 tablet_bootstrap.cc:654] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:55.836040   857 leader_election.cc:343] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:55.836508   857 leader_election.cc:304] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:55.837220  1542 raft_consensus.cc:2966] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:55.837795   857 leader_election.cc:304] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:55.838593  1542 raft_consensus.cc:3024] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:55.838940  1542 raft_consensus.cc:534] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:55.838755   857 leader_election.cc:304] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:55.839260  1542 raft_consensus.cc:3296] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.839881  1500 raft_consensus.cc:3024] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:55.840270  1500 raft_consensus.cc:534] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:55.839980  1542 raft_consensus.cc:556] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.840616  1500 raft_consensus.cc:3296] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.848965  1542 leader_election.cc:290] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:55.849762  1500 raft_consensus.cc:556] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.852140  1500 leader_election.cc:290] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:55.853037  1496 leader_election.cc:290] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:55.855343  1437 raft_consensus.cc:534] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:55.855824  1437 raft_consensus.cc:556] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.865785  1180 raft_consensus.cc:366] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.866685  1180 raft_consensus.cc:392] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.866964  1180 raft_consensus.cc:787] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:55.867683  1180 consensus_queue.cc:261] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:55.872548   411 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.124:38133, user_credentials={real_user=slave}} blocked reactor thread for 116719us
I20261120 03:00:55.875154  1437 leader_election.cc:290] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:55.876881  1050 tablet_service.cc:1495] Processing CreateTablet for tablet dc4133a8f33b4902a6308f9bdd4f91d2 (DEFAULT_TABLE table=default.Table_0c88eb71972c46cd918b4abf3ee01e11 [id=e24a1e3bb5054f6198ebd21469ab6f3c]), partition=RANGE (key) PARTITION UNBOUNDED
W20261120 03:00:55.884747   857 leader_election.cc:343] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:55.885494   857 leader_election.cc:343] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:55.892181  1180 ts_tablet_manager.cc:1480] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.058s	user 0.006s	sys 0.000s
I20261120 03:00:55.904454  1076 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "88b9e9692a874ae993dde4b9e2586a70" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:55.906198  1181 ts_tablet_manager.cc:1480] T d736d65c3e0b419084b4934dad83f16a P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.449s	user 0.009s	sys 0.000s
I20261120 03:00:55.906004  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b6f0ff2246cd4772af47812b699e93e8" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:55.907501  1181 tablet_bootstrap.cc:492] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:55.915584  1180 tablet_bootstrap.cc:492] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:55.912801  1520 raft_consensus.cc:534] T fbbe19dadb08455285078e239d81bcca P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:55.916198  1324 tablet_bootstrap.cc:492] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:55.916600  1324 ts_tablet_manager.cc:1449] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.088s	user 0.028s	sys 0.005s
W20261120 03:00:55.917665   857 leader_election.cc:343] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:55.916304  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0db7db25d779459ebde532f938493975" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:55.908274  1050 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet dc4133a8f33b4902a6308f9bdd4f91d2. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:00:55.919025   857 leader_election.cc:343] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:55.919066  1324 raft_consensus.cc:366] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.919628  1324 raft_consensus.cc:392] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.919888  1324 raft_consensus.cc:787] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:55.920528  1324 consensus_queue.cc:261] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.921208  1075 raft_consensus.cc:2062] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:55.916559  1069 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d736d65c3e0b419084b4934dad83f16a" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:55.922454  1069 raft_consensus.cc:2062] T d736d65c3e0b419084b4934dad83f16a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:55.922773  1324 ts_tablet_manager.cc:1480] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.006s	user 0.006s	sys 0.001s
I20261120 03:00:55.923595  1324 tablet_bootstrap.cc:492] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
W20261120 03:00:55.929657   411 leader_election.cc:336] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133): Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111)
I20261120 03:00:55.930186   411 leader_election.cc:304] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e102e07459954b4ea13b65ab9fd5d18b; no voters: 18331bef5c884dcab9dbd26bc2c82273, 2599fd9eef7d4ce990c40e2356a37c3f
I20261120 03:00:55.933157  1324 tablet_bootstrap.cc:654] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:55.916558  1520 raft_consensus.cc:556] T fbbe19dadb08455285078e239d81bcca P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:55.938218   709 leader_election.cc:343] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:55.939783  1496 consensus_queue.cc:1049] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:55.940303  1180 tablet_bootstrap.cc:654] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:55.949007  1324 tablet_bootstrap.cc:492] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:55.949457  1324 ts_tablet_manager.cc:1449] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.026s	user 0.010s	sys 0.000s
I20261120 03:00:55.951958  1324 raft_consensus.cc:366] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.952589  1324 raft_consensus.cc:392] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.952854  1324 raft_consensus.cc:787] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:55.953452  1324 consensus_queue.cc:261] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.954843  1496 raft_consensus.cc:534] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:55.955297  1496 raft_consensus.cc:556] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.956303  1324 ts_tablet_manager.cc:1480] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.007s	user 0.005s	sys 0.000s
I20261120 03:00:55.958664  1496 leader_election.cc:290] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:55.961365  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d736d65c3e0b419084b4934dad83f16a" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:55.961954  1074 raft_consensus.cc:3296] T d736d65c3e0b419084b4934dad83f16a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.962250  1180 tablet_bootstrap.cc:492] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:55.962714  1180 ts_tablet_manager.cc:1449] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.047s	user 0.006s	sys 0.007s
I20261120 03:00:55.966418  1180 raft_consensus.cc:366] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.967108  1180 raft_consensus.cc:392] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:55.967368  1180 raft_consensus.cc:787] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:55.968040  1180 consensus_queue.cc:261] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.970907   857 leader_election.cc:304] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:00:55.971891   857 leader_election.cc:304] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:00:55.975932   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "fbbe19dadb08455285078e239d81bcca" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:55.976742  1500 raft_consensus.cc:3024] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:55.977087  1500 raft_consensus.cc:534] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:55.977396  1500 raft_consensus.cc:3296] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.933246  1517 raft_consensus.cc:556] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.978209  1504 raft_consensus.cc:3024] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:55.978174  1500 raft_consensus.cc:556] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.978582  1504 raft_consensus.cc:534] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:55.978904  1504 raft_consensus.cc:3296] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:55.936919  1392 ts_tablet_manager.cc:1975] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:55.979821  1392 log.cc:1199] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/c3b56d0ccab140c28a37808d9d5902c6
I20261120 03:00:55.979648  1504 raft_consensus.cc:556] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:55.980786  1392 ts_tablet_manager.cc:1996] T c3b56d0ccab140c28a37808d9d5902c6 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:55.994693  1520 leader_election.cc:290] T fbbe19dadb08455285078e239d81bcca P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:56.002120   857 leader_election.cc:304] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:56.003975  1295 ts_tablet_manager.cc:1975] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:56.005282  1295 log.cc:1199] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/00dd2ffad1124244b171b2657d38a708
I20261120 03:00:56.006011  1295 ts_tablet_manager.cc:1996] T 00dd2ffad1124244b171b2657d38a708 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:56.007488  1548 raft_consensus.cc:2966] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
W20261120 03:00:56.013764   411 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.124:38133, user_credentials={real_user=slave}} blocked reactor thread for 84133.5us
I20261120 03:00:55.997983  1563 raft_consensus.cc:534] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:56.018090  1563 raft_consensus.cc:556] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.023144  1563 leader_election.cc:290] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:56.037012  1563 raft_consensus.cc:534] T daae644853734cd5818a46ecaa5ce7df P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:56.037576  1563 raft_consensus.cc:556] T daae644853734cd5818a46ecaa5ce7df P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.039595  1563 leader_election.cc:290] T daae644853734cd5818a46ecaa5ce7df P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:56.040280  1181 tablet_bootstrap.cc:654] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:56.015642   709 outbound_call.cc:321] RPC callback for RPC call kudu.master.MasterService.TSHeartbeat -> {remote=127.0.96.126:44467, user_credentials={real_user=slave}} blocked reactor thread for 72839.5us
I20261120 03:00:56.047662  1069 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a6ae8007455d4be3b50672b9c6b0fd8f" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:56.059414  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 75) took 2231 ms. Trace:
I20261120 03:00:56.059808  1392 rpcz_store.cc:276] 1120 03:00:53.828272 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.830111 (+  1839us) service_pool.cc:224] Handling call
1120 03:00:54.892221 (+1062110us) ts_tablet_manager.cc:1197] Deleting tablet c3b56d0ccab140c28a37808d9d5902c6
1120 03:00:54.892277 (+    56us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:55.139667 (+247390us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:55.609209 (+469542us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:56.059339 (+450130us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1061500}
I20261120 03:00:56.061188  1392 tablet_replica.cc:333] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:56.061985  1392 raft_consensus.cc:2440] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:56.062599  1392 pending_rounds.cc:70] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234: Trying to abort 1 pending ops.
I20261120 03:00:56.063014  1392 pending_rounds.cc:77] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7352908392629579776 op_type: NO_OP noop_request { }
I20261120 03:00:56.063483  1392 raft_consensus.cc:3115] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: NO_OP replication failed: Aborted: Op aborted
I20261120 03:00:56.063863  1392 raft_consensus.cc:2469] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:56.066051  1392 ts_tablet_manager.cc:1962] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
W20261120 03:00:56.066958   709 leader_election.cc:343] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:56.073776  1074 raft_consensus.cc:2062] T d736d65c3e0b419084b4934dad83f16a P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:56.077984  1295 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 74) took 2767 ms. Trace:
I20261120 03:00:56.078349  1295 rpcz_store.cc:276] 1120 03:00:53.310249 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.317260 (+  7011us) service_pool.cc:224] Handling call
1120 03:00:55.059754 (+1742494us) ts_tablet_manager.cc:1197] Deleting tablet 00dd2ffad1124244b171b2657d38a708
1120 03:00:55.059800 (+    46us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:55.519786 (+459986us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:56.003832 (+484046us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:56.077918 (+ 74086us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1658832}
I20261120 03:00:56.079180  1180 ts_tablet_manager.cc:1480] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.116s	user 0.005s	sys 0.002s
I20261120 03:00:56.079628  1295 tablet_replica.cc:333] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:56.080063  1180 tablet_bootstrap.cc:492] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:56.080221  1522 raft_consensus.cc:3024] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:56.107259  1045 tablet_service.cc:1495] Processing CreateTablet for tablet 091dd73d27df4db09ab750747da7584e (DEFAULT_TABLE table=default.Table_3c9436f0bbcd41bb987053b52acc02cf [id=b349c04100df4df599ad63dd3802bdd8]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.108453  1045 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 091dd73d27df4db09ab750747da7584e. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.108661  1048 tablet_service.cc:1495] Processing CreateTablet for tablet 014bd9c83be34c31b90fe4bcf31f1649 (DEFAULT_TABLE table=default.Table_cfa8989fb1134d908ab28bd6ebb80563 [id=16bb3f7e35c54a8c936b4af5a66a7f17]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.109699  1048 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 014bd9c83be34c31b90fe4bcf31f1649. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.110999  1046 tablet_service.cc:1495] Processing CreateTablet for tablet a04da9cb8f004443aa380902ea5427c5 (DEFAULT_TABLE table=default.Table_8fdb321b36394e5ea7df159be584695b [id=773d7fe26a094fd3872938cb518f36ad]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.111183  1059 tablet_service.cc:1495] Processing CreateTablet for tablet f376fdc03b3b4e3680171af6b8574c2e (DEFAULT_TABLE table=default.Table_7f80bbdf810749109360575194927d04 [id=382f9ae93f5e47baa2f361c7d853f5a9]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.106130   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet c3b56d0ccab140c28a37808d9d5902c6 (table default.Table_84d4c48133054235908348e9d8cf1f2a [id=5312622ebb1347f78430eab828632d38]) successfully deleted
I20261120 03:00:56.112643  1046 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a04da9cb8f004443aa380902ea5427c5. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.112571  1047 tablet_service.cc:1495] Processing CreateTablet for tablet b3f7103b59d443f9bc739083b8e023ce (DEFAULT_TABLE table=default.Table_555e4ecfadce486baefa8c3e2d1bc272 [id=def72d7bfc214aa5a942c18714c0173e]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.113569  1047 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b3f7103b59d443f9bc739083b8e023ce. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.126547   493 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_931d2c41a0834cf19b10a6d61e85698a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:56.128549   493 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_931d2c41a0834cf19b10a6d61e85698a in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:56.137107  1059 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f376fdc03b3b4e3680171af6b8574c2e. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.138226  1461 raft_consensus.cc:534] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:56.138680  1461 raft_consensus.cc:556] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.139402   751 tablet_service.cc:1495] Processing CreateTablet for tablet 091dd73d27df4db09ab750747da7584e (DEFAULT_TABLE table=default.Table_3c9436f0bbcd41bb987053b52acc02cf [id=b349c04100df4df599ad63dd3802bdd8]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.139918   750 tablet_service.cc:1495] Processing CreateTablet for tablet b3f7103b59d443f9bc739083b8e023ce (DEFAULT_TABLE table=default.Table_555e4ecfadce486baefa8c3e2d1bc272 [id=def72d7bfc214aa5a942c18714c0173e]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.140473   751 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 091dd73d27df4db09ab750747da7584e. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.140509   755 tablet_service.cc:1495] Processing CreateTablet for tablet f376fdc03b3b4e3680171af6b8574c2e (DEFAULT_TABLE table=default.Table_7f80bbdf810749109360575194927d04 [id=382f9ae93f5e47baa2f361c7d853f5a9]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.141462   753 tablet_service.cc:1495] Processing CreateTablet for tablet a04da9cb8f004443aa380902ea5427c5 (DEFAULT_TABLE table=default.Table_8fdb321b36394e5ea7df159be584695b [id=773d7fe26a094fd3872938cb518f36ad]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.142194   756 tablet_service.cc:1495] Processing CreateTablet for tablet 014bd9c83be34c31b90fe4bcf31f1649 (DEFAULT_TABLE table=default.Table_cfa8989fb1134d908ab28bd6ebb80563 [id=16bb3f7e35c54a8c936b4af5a66a7f17]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.141455   755 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f376fdc03b3b4e3680171af6b8574c2e. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.143370   756 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 014bd9c83be34c31b90fe4bcf31f1649. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.143978   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a04da9cb8f004443aa380902ea5427c5. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.143236   775 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d736d65c3e0b419084b4934dad83f16a" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:56.148607   771 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "daae644853734cd5818a46ecaa5ce7df" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:56.153928  1573 consensus_queue.cc:1049] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20261120 03:00:56.151322   772 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0db7db25d779459ebde532f938493975" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:56.154906   772 raft_consensus.cc:2062] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:00:56.140563  1461 leader_election.cc:290] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:56.157867   710 leader_election.cc:304] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:56.158255  1504 leader_election.cc:290] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:56.171393   773 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "88b9e9692a874ae993dde4b9e2586a70" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:56.172166   773 raft_consensus.cc:3296] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:56.173038   757 tablet_service.cc:1495] Processing CreateTablet for tablet 600a18b7e5104a78935d906b8adfde6f (DEFAULT_TABLE table=default.Table_d1737969bdd644a6ab39da121eeedd60 [id=ed325a58ff124ba9a6b1d05bc901793b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.188309   775 raft_consensus.cc:2628] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 5beeefe8bef54caeac463d05d8c30fa6 in this term.
I20261120 03:00:56.189189  1555 raft_consensus.cc:3024] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:56.189568  1555 raft_consensus.cc:534] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:56.189916  1555 raft_consensus.cc:3296] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:56.194849  1522 raft_consensus.cc:744] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:00:56.174698   750 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b3f7103b59d443f9bc739083b8e023ce. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.205638  1069 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3d89fb1d7fa14ae8ac252c68338ed4d6" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:56.206948   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0db7db25d779459ebde532f938493975" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:56.207650   776 raft_consensus.cc:2062] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:56.195931  1522 consensus_queue.cc:238] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:56.214354   774 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b6f0ff2246cd4772af47812b699e93e8" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:56.215818   752 tablet_service.cc:1495] Processing CreateTablet for tablet c6a65c1a4a574961bec657ad7c9048e0 (DEFAULT_TABLE table=default.Table_dd2f7617ff0e401b9b862d5991110dd5 [id=3955f41b6d4d41bf8ba2c81e4ac7f4a8]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.219143  1051 tablet_service.cc:1495] Processing CreateTablet for tablet 600a18b7e5104a78935d906b8adfde6f (DEFAULT_TABLE table=default.Table_d1737969bdd644a6ab39da121eeedd60 [id=ed325a58ff124ba9a6b1d05bc901793b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.219463  1075 raft_consensus.cc:3296] T daae644853734cd5818a46ecaa5ce7df P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:56.222929  1324 tablet_bootstrap.cc:492] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:56.226686   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d736d65c3e0b419084b4934dad83f16a" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:56.227468   920 raft_consensus.cc:2628] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 5beeefe8bef54caeac463d05d8c30fa6 in current term 1: Already voted for candidate 3d30c2428730481db9b0ffd279dd3440 in this term.
I20261120 03:00:56.228809   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 00dd2ffad1124244b171b2657d38a708 (table default.Table_e07b2ba4d9864d148d7088622b10e03c [id=201bf626c7d643d7a98a680b1e714ba3]) successfully deleted
I20261120 03:00:56.230849   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a6ae8007455d4be3b50672b9c6b0fd8f" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:56.232295  1295 raft_consensus.cc:2440] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:56.232925  1180 tablet_bootstrap.cc:654] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:56.232968  1295 pending_rounds.cc:70] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440: Trying to abort 1 pending ops.
I20261120 03:00:56.233244  1295 pending_rounds.cc:77] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7352908413213188096 op_type: NO_OP noop_request { }
I20261120 03:00:56.233557  1295 raft_consensus.cc:3115] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
I20261120 03:00:56.233860  1295 raft_consensus.cc:2469] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:56.236768  1295 ts_tablet_manager.cc:1962] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:56.242061   901 tablet_service.cc:1495] Processing CreateTablet for tablet 600a18b7e5104a78935d906b8adfde6f (DEFAULT_TABLE table=default.Table_d1737969bdd644a6ab39da121eeedd60 [id=ed325a58ff124ba9a6b1d05bc901793b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.224298   773 raft_consensus.cc:2062] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:00:56.243213   901 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 600a18b7e5104a78935d906b8adfde6f. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.245966  1011 leader_election.cc:304] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:00:56.250681   902 tablet_service.cc:1495] Processing CreateTablet for tablet c6a65c1a4a574961bec657ad7c9048e0 (DEFAULT_TABLE table=default.Table_dd2f7617ff0e401b9b862d5991110dd5 [id=3955f41b6d4d41bf8ba2c81e4ac7f4a8]), partition=RANGE (key) PARTITION UNBOUNDED
W20261120 03:00:56.250941   710 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 74672.6us
I20261120 03:00:56.252918   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c6a65c1a4a574961bec657ad7c9048e0. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:00:56.254194   709 leader_election.cc:343] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:56.257929   752 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c6a65c1a4a574961bec657ad7c9048e0. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.263615   757 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 600a18b7e5104a78935d906b8adfde6f. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.235051   921 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3d89fb1d7fa14ae8ac252c68338ed4d6" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:56.269210   899 tablet_service.cc:1495] Processing CreateTablet for tablet a04da9cb8f004443aa380902ea5427c5 (DEFAULT_TABLE table=default.Table_8fdb321b36394e5ea7df159be584695b [id=773d7fe26a094fd3872938cb518f36ad]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.275981  1578 raft_consensus.cc:534] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:56.276870  1578 raft_consensus.cc:556] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.274487   771 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d736d65c3e0b419084b4934dad83f16a" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:56.284770   771 raft_consensus.cc:2628] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 5beeefe8bef54caeac463d05d8c30fa6 in this term.
I20261120 03:00:56.291913  1578 leader_election.cc:290] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:56.294414  1324 tablet_bootstrap.cc:654] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:56.297071  1381 consensus_meta.cc:298] T a1f6197c60dd4dfb8aeb5c38373aab21Time spent flushing consensus metadata: real 1.030s	user 0.008s	sys 0.000s
I20261120 03:00:56.298655  1051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 600a18b7e5104a78935d906b8adfde6f. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:00:56.318675   710 leader_election.cc:343] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:56.319069   710 leader_election.cc:304] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
W20261120 03:00:56.320163   710 leader_election.cc:343] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:56.320477   710 leader_election.cc:304] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:56.321079  1072 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.96.65:39215 (request call id 85) took 1082 ms. Trace:
I20261120 03:00:56.321448  1072 rpcz_store.cc:276] 1120 03:00:55.238811 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:55.265595 (+ 26784us) service_pool.cc:224] Handling call
1120 03:00:55.266017 (+   422us) raft_consensus.cc:1536] Updating replica for 1 ops
1120 03:00:55.294295 (+ 28278us) raft_consensus.cc:1602] Early marking committed up to index 0
1120 03:00:55.294331 (+    36us) raft_consensus.cc:1607] Triggering prepare for 1 ops
1120 03:00:55.295076 (+   745us) log.cc:844] Serialized 26 byte log entry
1120 03:00:55.707806 (+412730us) raft_consensus.cc:1713] Marking committed up to 1
1120 03:00:55.708361 (+   555us) log.cc:844] Serialized 14 byte log entry
1120 03:00:55.833697 (+125336us) raft_consensus.cc:1765] Filling consensus response to leader.
1120 03:00:55.871038 (+ 37341us) raft_consensus.cc:1736] Waiting on the replicates to finish logging
1120 03:00:55.882102 (+ 11064us) raft_consensus.cc:1753] finished
1120 03:00:55.892620 (+ 10518us) raft_consensus.cc:1759] UpdateReplicas() finished
1120 03:00:56.320891 (+428271us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":1805,"prepare.queue_time_us":18866,"prepare.run_cpu_time_us":31,"prepare.run_wall_time_us":32,"raft.queue_time_us":204982,"raft.run_cpu_time_us":9216,"raft.run_wall_time_us":1046628,"spinlock_wait_cycles":19328,"thread_start_us":437221,"threads_started":2,"wal-append.queue_time_us":159843}
I20261120 03:00:56.323421   921 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "fbbe19dadb08455285078e239d81bcca" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:56.325332   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0db7db25d779459ebde532f938493975" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:56.326025   920 raft_consensus.cc:2628] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 823abdbd90ba47bbacad0f8dcff78234 in current term 1: Already voted for candidate 3d30c2428730481db9b0ffd279dd3440 in this term.
I20261120 03:00:56.325368   917 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "168cb98bb1b242a5a0377bf292204d96" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:56.326699   898 tablet_service.cc:1495] Processing CreateTablet for tablet a1192a7f050a4f5387a30b2216dec5f0 (DEFAULT_TABLE table=default.Table_646d629dfeea413a86bc3edf68c7d6ae [id=aa0b779e8ea84d928af40806a7913f1a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.327271   917 raft_consensus.cc:2062] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:56.328161  1578 raft_consensus.cc:2966] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:56.328351  1461 raft_consensus.cc:2966] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:56.328825   922 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "daae644853734cd5818a46ecaa5ce7df" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:56.329418  1461 raft_consensus.cc:534] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:56.329536  1578 consensus_queue.cc:1049] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:56.329864  1461 raft_consensus.cc:556] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.331681  1461 leader_election.cc:290] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:56.333482  1049 tablet_service.cc:1495] Processing CreateTablet for tablet c6a65c1a4a574961bec657ad7c9048e0 (DEFAULT_TABLE table=default.Table_dd2f7617ff0e401b9b862d5991110dd5 [id=3955f41b6d4d41bf8ba2c81e4ac7f4a8]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.334347  1057 tablet_service.cc:1495] Processing CreateTablet for tablet a1192a7f050a4f5387a30b2216dec5f0 (DEFAULT_TABLE table=default.Table_646d629dfeea413a86bc3edf68c7d6ae [id=aa0b779e8ea84d928af40806a7913f1a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.335408  1012 leader_election.cc:304] T fbbe19dadb08455285078e239d81bcca P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:56.335482   709 leader_election.cc:304] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440
I20261120 03:00:56.337186  1553 raft_consensus.cc:3024] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
W20261120 03:00:56.337339  1569 log.cc:927] Time spent T fbbe19dadb08455285078e239d81bcca P 823abdbd90ba47bbacad0f8dcff78234: Append to log took a long time: real 0.105s	user 0.001s	sys 0.000s
I20261120 03:00:56.338621  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "168cb98bb1b242a5a0377bf292204d96" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:56.339686   710 leader_election.cc:304] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:56.340257  1553 raft_consensus.cc:744] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:56.340853   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "c6fea104598e4f6d876f96675f2da816" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:56.347200  1574 raft_consensus.cc:534] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:56.347744  1574 raft_consensus.cc:556] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.348829  1496 raft_consensus.cc:3024] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:56.351137   923 raft_consensus.cc:2062] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:56.350975  1012 leader_election.cc:304] T daae644853734cd5818a46ecaa5ce7df P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:56.352228   710 leader_election.cc:304] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:56.330796  1076 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "d736d65c3e0b419084b4934dad83f16a" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:56.353114  1076 raft_consensus.cc:2628] T d736d65c3e0b419084b4934dad83f16a P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 5beeefe8bef54caeac463d05d8c30fa6 in this term.
I20261120 03:00:56.350414  1496 raft_consensus.cc:534] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:56.354442   857 leader_election.cc:304] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:56.355145  1504 raft_consensus.cc:2966] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20261120 03:00:56.354656  1554 raft_consensus.cc:3024] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:56.362920  1392 ts_tablet_manager.cc:1975] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:56.363291  1392 log.cc:1199] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/06c3bac919e342b09dd6af13aebc387b
I20261120 03:00:56.363988  1392 ts_tablet_manager.cc:1996] T 06c3bac919e342b09dd6af13aebc387b P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:56.365978  1057 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a1192a7f050a4f5387a30b2216dec5f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.366971  1554 raft_consensus.cc:534] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:56.367308  1554 raft_consensus.cc:3296] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:56.368019  1554 raft_consensus.cc:556] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.354722  1496 raft_consensus.cc:3296] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:56.369370  1049 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c6a65c1a4a574961bec657ad7c9048e0. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.369292  1496 raft_consensus.cc:556] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.345132  1553 consensus_queue.cc:238] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.376130   857 leader_election.cc:304] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:56.377111  1574 leader_election.cc:290] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:56.377595  1574 raft_consensus.cc:3024] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:56.378592   773 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "c6fea104598e4f6d876f96675f2da816" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:56.379390   773 raft_consensus.cc:2628] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 5beeefe8bef54caeac463d05d8c30fa6 in this term.
I20261120 03:00:56.379303  1500 leader_election.cc:290] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:56.380910   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0db7db25d779459ebde532f938493975" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:56.381831   776 raft_consensus.cc:3296] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:56.382892   776 raft_consensus.cc:2062] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:00:56.386655  1574 raft_consensus.cc:744] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
W20261120 03:00:56.388404   709 leader_election.cc:343] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:56.388227  1574 consensus_queue.cc:238] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.393890  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0db7db25d779459ebde532f938493975" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:56.394083   754 tablet_service.cc:1495] Processing CreateTablet for tablet a1192a7f050a4f5387a30b2216dec5f0 (DEFAULT_TABLE table=default.Table_646d629dfeea413a86bc3edf68c7d6ae [id=aa0b779e8ea84d928af40806a7913f1a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.395226   754 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a1192a7f050a4f5387a30b2216dec5f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.398840  1181 tablet_bootstrap.cc:492] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:56.399279  1181 ts_tablet_manager.cc:1449] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.492s	user 0.013s	sys 0.005s
I20261120 03:00:56.407862  1076 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "c6fea104598e4f6d876f96675f2da816" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:56.410898   857 leader_election.cc:343] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:56.411274   857 leader_election.cc:304] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:56.412581  1181 raft_consensus.cc:366] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.413028  1068 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "c6fea104598e4f6d876f96675f2da816" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:56.413373  1181 raft_consensus.cc:392] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:56.413640  1181 raft_consensus.cc:787] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:56.414278  1181 consensus_queue.cc:261] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.416394  1181 ts_tablet_manager.cc:1480] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.017s	user 0.003s	sys 0.002s
I20261120 03:00:56.416415  1555 raft_consensus.cc:556] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.417601   774 raft_consensus.cc:2628] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 5beeefe8bef54caeac463d05d8c30fa6 in this term.
I20261120 03:00:56.429162  1324 tablet_bootstrap.cc:492] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:56.429551  1324 ts_tablet_manager.cc:1449] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.207s	user 0.007s	sys 0.011s
I20261120 03:00:56.431799  1324 raft_consensus.cc:366] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.432399  1324 raft_consensus.cc:392] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:56.432644  1324 raft_consensus.cc:787] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:56.433203  1324 consensus_queue.cc:261] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:56.438999   709 leader_election.cc:343] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:56.444386  1517 leader_election.cc:290] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:56.446796  1567 raft_consensus.cc:3024] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:56.447300  1567 raft_consensus.cc:534] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:56.447710  1567 raft_consensus.cc:3296] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:56.448436  1504 raft_consensus.cc:3296] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:56.448818   899 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a04da9cb8f004443aa380902ea5427c5. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.449208  1504 raft_consensus.cc:2966] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:56.449811   898 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet a1192a7f050a4f5387a30b2216dec5f0. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:00:56.457768   710 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 51000.1us
I20261120 03:00:56.458947   857 leader_election.cc:304] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:56.472084  1555 leader_election.cc:290] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:56.474052  1549 raft_consensus.cc:2966] T daae644853734cd5818a46ecaa5ce7df P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:56.458160   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ce612f9a121b4b18908724d2726e156f" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:56.476979  1381 raft_consensus.cc:2966] T fbbe19dadb08455285078e239d81bcca P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20261120 03:00:56.474857   923 raft_consensus.cc:3296] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:56.473106  1437 raft_consensus.cc:2966] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
W20261120 03:00:56.499560  1011 leader_election.cc:343] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: ce612f9a121b4b18908724d2726e156f
I20261120 03:00:56.509130   922 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b6f0ff2246cd4772af47812b699e93e8" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:56.509359  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 77) took 2679 ms. Trace:
I20261120 03:00:56.509974   922 raft_consensus.cc:2628] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 5beeefe8bef54caeac463d05d8c30fa6 in current term 1: Already voted for candidate 3d30c2428730481db9b0ffd279dd3440 in this term.
I20261120 03:00:56.449477  1567 raft_consensus.cc:556] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.511672   923 raft_consensus.cc:2062] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
I20261120 03:00:56.514154  1180 tablet_bootstrap.cc:492] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:56.514545  1180 ts_tablet_manager.cc:1449] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.435s	user 0.011s	sys 0.008s
I20261120 03:00:56.516196   756 tablet_service.cc:1495] Processing CreateTablet for tablet 9dadc05b726346309d6b350f9abb4abb (DEFAULT_TABLE table=default.Table_027fdfc1742a404ea34dfdc00e41ff5a [id=7ec945bb5372463b954d8e5d67879627]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.517321   756 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9dadc05b726346309d6b350f9abb4abb. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.517889  1180 raft_consensus.cc:366] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.518599  1180 raft_consensus.cc:392] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:56.518857  1180 raft_consensus.cc:787] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:56.519060  1479 consensus_queue.cc:1049] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:56.519539  1180 consensus_queue.cc:261] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.527362  1180 ts_tablet_manager.cc:1480] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.013s	user 0.005s	sys 0.000s
I20261120 03:00:56.531085  1180 tablet_bootstrap.cc:492] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:56.536481  1074 raft_consensus.cc:2628] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 823abdbd90ba47bbacad0f8dcff78234 in this term.
I20261120 03:00:56.545689  1485 raft_consensus.cc:534] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:56.546243  1485 raft_consensus.cc:556] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.548074  1485 leader_election.cc:290] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:56.548640  1485 raft_consensus.cc:534] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:56.549000  1485 raft_consensus.cc:556] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.551122  1485 leader_election.cc:290] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:56.556078  1555 raft_consensus.cc:534] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:56.556565  1555 raft_consensus.cc:556] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.558531  1555 leader_election.cc:290] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:56.563750  1496 leader_election.cc:290] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:56.565193  1050 tablet_service.cc:1495] Processing CreateTablet for tablet 9dadc05b726346309d6b350f9abb4abb (DEFAULT_TABLE table=default.Table_027fdfc1742a404ea34dfdc00e41ff5a [id=7ec945bb5372463b954d8e5d67879627]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.509652  1392 rpcz_store.cc:276] 1120 03:00:53.830421 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.831045 (+   624us) service_pool.cc:224] Handling call
1120 03:00:56.060999 (+2229954us) ts_tablet_manager.cc:1197] Deleting tablet 06c3bac919e342b09dd6af13aebc387b
1120 03:00:56.061072 (+    73us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:56.260534 (+199462us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:56.362792 (+102258us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:56.509301 (+146509us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2229290}
I20261120 03:00:56.578985  1295 ts_tablet_manager.cc:1975] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:56.579370  1295 log.cc:1199] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/1885397e42774a89bb347bc776284bc0
I20261120 03:00:56.580039  1295 ts_tablet_manager.cc:1996] T 1885397e42774a89bb347bc776284bc0 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:56.580924  1295 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 78) took 3225 ms. Trace:
I20261120 03:00:56.581198  1295 rpcz_store.cc:276] 1120 03:00:53.355460 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.356085 (+   625us) service_pool.cc:224] Handling call
1120 03:00:56.079481 (+2723396us) ts_tablet_manager.cc:1197] Deleting tablet 1885397e42774a89bb347bc776284bc0
1120 03:00:56.079522 (+    41us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:56.438714 (+359192us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:56.578854 (+140140us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:56.580871 (+  2017us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":151882,"tablet-delete.queue_time_us":2673639}
I20261120 03:00:56.583767  1295 tablet_replica.cc:333] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:56.584542  1295 raft_consensus.cc:2440] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:56.584981  1295 raft_consensus.cc:2469] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:56.587795   753 tablet_service.cc:1495] Processing CreateTablet for tablet b549a06d15db449b851fe9549a963416 (DEFAULT_TABLE table=default.Table_61cb1e03b7e442a8822e32a2295866f3 [id=b5b565c2c2224675baada2ae9b42be54]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.588800   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b549a06d15db449b851fe9549a963416. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.582183  1050 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9dadc05b726346309d6b350f9abb4abb. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.601610  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5c5c0ac869a1419fad9f61ec647027ee" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:56.603240  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "dc4133a8f33b4902a6308f9bdd4f91d2" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:56.606911  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a08aa5ddcf1a4061a70275a8eecb1856" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:56.609371  1567 leader_election.cc:290] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:56.614352  1324 ts_tablet_manager.cc:1480] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.185s	user 0.018s	sys 0.011s
I20261120 03:00:56.615348  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "168cb98bb1b242a5a0377bf292204d96" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:56.621524  1324 tablet_bootstrap.cc:492] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
W20261120 03:00:56.622150   709 leader_election.cc:343] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:56.623574   709 leader_election.cc:343] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:56.626286   709 leader_election.cc:343] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:56.627610   709 leader_election.cc:343] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:56.629796  1055 tablet_service.cc:1495] Processing CreateTablet for tablet b549a06d15db449b851fe9549a963416 (DEFAULT_TABLE table=default.Table_61cb1e03b7e442a8822e32a2295866f3 [id=b5b565c2c2224675baada2ae9b42be54]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.630990  1055 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b549a06d15db449b851fe9549a963416. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:00:56.638703   486 consensus_peers.cc:597] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 -> Peer 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133): Couldn't send request to peer 2599fd9eef7d4ce990c40e2356a37c3f. Status: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20261120 03:00:56.648604  1554 leader_election.cc:290] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:56.649078  1324 tablet_bootstrap.cc:654] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:56.655588   774 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0db7db25d779459ebde532f938493975" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:56.656363   774 raft_consensus.cc:2628] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 823abdbd90ba47bbacad0f8dcff78234 in current term 1: Already voted for candidate 3d30c2428730481db9b0ffd279dd3440 in this term.
I20261120 03:00:56.682317  1324 tablet_bootstrap.cc:492] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:56.682762  1324 ts_tablet_manager.cc:1449] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.061s	user 0.011s	sys 0.003s
I20261120 03:00:56.685192  1324 raft_consensus.cc:366] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.685810  1324 raft_consensus.cc:392] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:56.686045  1324 raft_consensus.cc:787] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:56.694927  1324 consensus_queue.cc:261] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.700985  1324 ts_tablet_manager.cc:1480] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.015s	user 0.006s	sys 0.000s
I20261120 03:00:56.701954  1324 tablet_bootstrap.cc:492] T 014bd9c83be34c31b90fe4bcf31f1649 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:56.708235  1181 tablet_bootstrap.cc:492] T 68179488b9bc4e03b65a9f1d434c95b5 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:56.735252  1324 tablet_bootstrap.cc:654] T 014bd9c83be34c31b90fe4bcf31f1649 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:56.748255   495 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 68179488b9bc4e03b65a9f1d434c95b5
I20261120 03:00:56.749768   495 catalog_manager.cc:6952] ReplaceTablet: tablet 68179488b9bc4e03b65a9f1d434c95b5 deleted and replaced by tablet 4d79fc4c8f1c4a5f83c0dc94337c857e
I20261120 03:00:56.750983   495 rpcz_store.cc:275] Call kudu.master.MasterService.ReplaceTablet from 127.0.0.1:45640 (request call id 143) took 2135 ms. Trace:
I20261120 03:00:56.751294   495 rpcz_store.cc:276] 1120 03:00:54.615337 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.615496 (+   159us) service_pool.cc:224] Handling call
1120 03:00:56.750905 (+2135409us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:56.122325 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:56.122743 (+   418us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:56.122772 (+    29us) write_op.cc:440] Acquired schema lock
1120 03:00:56.122791 (+    19us) tablet.cc:710] Decoding operations
1120 03:00:56.123159 (+   368us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:56.123210 (+    51us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:56.123241 (+    31us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:00:56.123452 (+   211us) tablet.cc:758] Row locks acquired
1120 03:00:56.123463 (+    11us) write_op.cc:260] PREPARE: finished
1120 03:00:56.123580 (+   117us) write_op.cc:270] Start()
1120 03:00:56.123667 (+    87us) write_op.cc:276] Timestamp: P: 1795143656123442 usec, L: 0
1120 03:00:56.123680 (+    13us) op_driver.cc:350] REPLICATION: starting
1120 03:00:56.124142 (+   462us) log.cc:844] Serialized 622 byte log entry
1120 03:00:56.738966 (+614824us) op_driver.cc:466] REPLICATION: finished
1120 03:00:56.742982 (+  4016us) write_op.cc:301] APPLY: starting
1120 03:00:56.743044 (+    62us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:56.743150 (+   106us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:56.743251 (+   101us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:56.743622 (+   371us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:56.743811 (+   189us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:56.743852 (+    41us) write_op.cc:312] APPLY: finished
1120 03:00:56.744034 (+   182us) log.cc:844] Serialized 29 byte log entry
1120 03:00:56.745884 (+  1850us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:56.745950 (+    66us) write_op.cc:459] Released schema lock
1120 03:00:56.745980 (+    30us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"child_traces":[["op",{"apply.queue_time_us":3834,"num_ops":2,"prepare.queue_time_us":795009,"prepare.run_cpu_time_us":2041,"prepare.run_wall_time_us":2050,"replication_time_us":615180,"thread_start_us":3538,"threads_started":1}]]}
I20261120 03:00:56.754362   922 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5c5c0ac869a1419fad9f61ec647027ee" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:56.755119   922 raft_consensus.cc:2062] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:56.759466   921 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0db7db25d779459ebde532f938493975" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:56.759485   752 tablet_service.cc:1542] Processing DeleteTablet for tablet 68179488b9bc4e03b65a9f1d434c95b5 with delete_type TABLET_DATA_DELETED (replaced by tablet 4d79fc4c8f1c4a5f83c0dc94337c857e) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:56.758966   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a08aa5ddcf1a4061a70275a8eecb1856" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:56.760151   920 raft_consensus.cc:2062] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:00:56.760303   917 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "dc4133a8f33b4902a6308f9bdd4f91d2" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:56.771641  1324 tablet_bootstrap.cc:492] T 014bd9c83be34c31b90fe4bcf31f1649 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:56.772063  1324 ts_tablet_manager.cc:1449] T 014bd9c83be34c31b90fe4bcf31f1649 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.070s	user 0.011s	sys 0.000s
I20261120 03:00:56.774910  1324 raft_consensus.cc:366] T 014bd9c83be34c31b90fe4bcf31f1649 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.775523  1324 raft_consensus.cc:392] T 014bd9c83be34c31b90fe4bcf31f1649 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:56.775765  1324 raft_consensus.cc:787] T 014bd9c83be34c31b90fe4bcf31f1649 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:56.782219   497 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet a6ae8007455d4be3b50672b9c6b0fd8f
I20261120 03:00:56.784168   497 catalog_manager.cc:6952] ReplaceTablet: tablet a6ae8007455d4be3b50672b9c6b0fd8f deleted and replaced by tablet 8f1fda857e6c400e80d3d177a3d748ff
I20261120 03:00:56.784624   497 rpcz_store.cc:275] Call kudu.master.MasterService.ReplaceTablet from 127.0.0.1:45640 (request call id 149) took 1939 ms. Trace:
I20261120 03:00:56.784744   753 tablet_service.cc:1542] Processing DeleteTablet for tablet a6ae8007455d4be3b50672b9c6b0fd8f with delete_type TABLET_DATA_DELETED (replaced by tablet 8f1fda857e6c400e80d3d177a3d748ff) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:56.760164   921 raft_consensus.cc:2628] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 823abdbd90ba47bbacad0f8dcff78234 in current term 1: Already voted for candidate 3d30c2428730481db9b0ffd279dd3440 in this term.
I20261120 03:00:56.784934   497 rpcz_store.cc:276] 1120 03:00:54.844833 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.847542 (+  2709us) service_pool.cc:224] Handling call
1120 03:00:56.784543 (+1937001us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:56.124487 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:56.124907 (+   420us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:56.124929 (+    22us) write_op.cc:440] Acquired schema lock
1120 03:00:56.124940 (+    11us) tablet.cc:710] Decoding operations
1120 03:00:56.125267 (+   327us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:56.125300 (+    33us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:56.125319 (+    19us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:00:56.125508 (+   189us) tablet.cc:758] Row locks acquired
1120 03:00:56.125520 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:00:56.125620 (+   100us) write_op.cc:270] Start()
1120 03:00:56.125693 (+    73us) write_op.cc:276] Timestamp: P: 1795143656125491 usec, L: 0
1120 03:00:56.125708 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:00:56.182348 (+ 56640us) log.cc:844] Serialized 622 byte log entry
1120 03:00:56.742860 (+560512us) op_driver.cc:466] REPLICATION: finished
1120 03:00:56.757796 (+ 14936us) write_op.cc:301] APPLY: starting
1120 03:00:56.757855 (+    59us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:56.757965 (+   110us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:56.758063 (+    98us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:56.765657 (+  7594us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:56.765854 (+   197us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:56.765899 (+    45us) write_op.cc:312] APPLY: finished
1120 03:00:56.776638 (+ 10739us) log.cc:844] Serialized 29 byte log entry
1120 03:00:56.780977 (+  4339us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:56.781047 (+    70us) write_op.cc:459] Released schema lock
1120 03:00:56.781081 (+    34us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"child_traces":[["op",{"apply.queue_time_us":14787,"num_ops":2,"prepare.queue_time_us":779379,"prepare.run_cpu_time_us":2042,"prepare.run_wall_time_us":58121,"replication_time_us":617052,"threads_started":1}]]}
I20261120 03:00:56.777819   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "168cb98bb1b242a5a0377bf292204d96" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:56.791249   918 raft_consensus.cc:3296] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:56.795385  1324 consensus_queue.cc:261] T 014bd9c83be34c31b90fe4bcf31f1649 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.811888   921 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "c6fea104598e4f6d876f96675f2da816" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:56.812934   921 raft_consensus.cc:2062] T c6fea104598e4f6d876f96675f2da816 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:56.811192   496 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 014bd9c83be34c31b90fe4bcf31f1649
I20261120 03:00:56.814412   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 1885397e42774a89bb347bc776284bc0 (table default.Table_61cb1e03b7e442a8822e32a2295866f3 [id=b5b565c2c2224675baada2ae9b42be54]) successfully deleted
I20261120 03:00:56.815362   496 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:45640 (request call id 147) took 2084 ms. Trace:
I20261120 03:00:56.820624   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 5fee8dff291c43e9be0fb10b308718ce (table default.Table_a1f866996679431e9e1c4c5f6f8cdd12 [id=a275e215155b4824a1494f0954577fe8]). Expected version 1 got 0
I20261120 03:00:56.815685   496 rpcz_store.cc:276] 1120 03:00:54.730974 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.780000 (+ 49026us) service_pool.cc:224] Handling call
1120 03:00:54.806712 (+ 26712us) spinlock_profiling.cc:245] Waited 25.7 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07f2b6 00007f155abfbcf4 00007f155abfbc82 00007f155abfbbb1 00007f155abfbb2c 00007f155abfade2 00007f1558e990bc 00007f1558e984f6
1120 03:00:54.807689 (+   977us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:00:54.817246 (+  9557us) catalog_manager.cc:2859] Modifying in-memory table state
1120 03:00:54.817367 (+   121us) catalog_manager.cc:2868] Locking tablets
1120 03:00:55.318715 (+501348us) catalog_manager.cc:2882] Removing table and tablets from system table
1120 03:00:56.810709 (+1491994us) catalog_manager.cc:2900] Removing table from by-name map
1120 03:00:56.810911 (+   202us) catalog_manager.cc:2916] Committing in-memory state
1120 03:00:56.811000 (+    89us) catalog_manager.cc:2920] Aborting table tasks
1120 03:00:56.815256 (+  4256us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:55.338431 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:55.338852 (+   421us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:55.338878 (+    26us) write_op.cc:440] Acquired schema lock
1120 03:00:55.338891 (+    13us) tablet.cc:710] Decoding operations
1120 03:00:55.339237 (+   346us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:55.339277 (+    40us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:55.339303 (+    26us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:00:55.339603 (+   300us) tablet.cc:758] Row locks acquired
1120 03:00:55.339618 (+    15us) write_op.cc:260] PREPARE: finished
1120 03:00:55.339747 (+   129us) write_op.cc:270] Start()
1120 03:00:55.339832 (+    85us) write_op.cc:276] Timestamp: P: 1795143655339680 usec, L: 0
1120 03:00:55.339847 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:00:56.114373 (+774526us) spinlock_profiling.cc:245] Waited 513 ms on lock 0x7b5400030d98. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155bfabca4 00007f155b199da9 00007f155b155971 00007f155b15564a 00007f155b1dab55 00007f155b1daff9 00007f155b68b381 00007f155b68ab58 00007f155b68f7f1 00007f155b68f7a9 00007f155b68f739
1120 03:00:56.114669 (+   296us) log.cc:844] Serialized 725 byte log entry
1120 03:00:56.115720 (+  1051us) spinlock_profiling.cc:245] Waited 246 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b147a41 00007f155b1db018 00007f155b68b381 00007f155b68ab58 00007f155b68f7f1 00007f155b68f7a9 00007f155b68f739 00007f155b68f701 00007f155b68e92d 0000000000327654 0000000000327479
1120 03:00:56.696701 (+580981us) op_driver.cc:466] REPLICATION: finished
1120 03:00:56.739016 (+ 42315us) write_op.cc:301] APPLY: starting
1120 03:00:56.739089 (+    73us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:56.739223 (+   134us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:56.739241 (+    18us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:56.740457 (+  1216us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:56.740615 (+   158us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
1120 03:00:56.740662 (+    47us) write_op.cc:312] APPLY: finished
1120 03:00:56.740885 (+   223us) log.cc:844] Serialized 29 byte log entry
1120 03:00:56.810368 (+ 69483us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:56.810435 (+    67us) write_op.cc:459] Released schema lock
1120 03:00:56.810469 (+    34us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"spinlock_wait_cycles":59071744,"child_traces":[["op",{"apply.queue_time_us":42157,"num_ops":2,"prepare.queue_time_us":17821,"prepare.run_cpu_time_us":9112,"prepare.run_wall_time_us":777423,"replication_time_us":1356731,"spinlock_wait_cycles":1747099136,"thread_start_us":41877,"threads_started":1}]]}
I20261120 03:00:56.824618   498 catalog_manager.cc:5856] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66). New cstate: current_term: 1 leader_uuid: "3d30c2428730481db9b0ffd279dd3440" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: UNKNOWN } } }
I20261120 03:00:56.826269  1522 raft_consensus.cc:534] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:56.829362   493 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:45640 (request call id 153) took 1779 ms. Trace:
I20261120 03:00:56.829828   493 rpcz_store.cc:276] 1120 03:00:55.049523 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:55.597008 (+547485us) service_pool.cc:224] Handling call
1120 03:00:56.122045 (+525037us) spinlock_profiling.cc:245] Waited 518 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07efa6 00007f155abf71f4 00007f155abf7182 00007f155abf70b1 00007f155abf702c 00007f155abf62e2 00007f1558e990bc 00007f1558e984f6
1120 03:00:56.128986 (+  6941us) catalog_manager.cc:2145] Acquired catalog manager lock
1120 03:00:56.160572 (+ 31586us) catalog_manager.cc:2191] Created new table and tablet info
1120 03:00:56.828628 (+668056us) catalog_manager.cc:2251] Wrote table and tablets to system table
1120 03:00:56.828955 (+   327us) catalog_manager.cc:2285] Inserted table and tablets into CatalogManager maps
1120 03:00:56.829271 (+   316us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:56.235855 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:56.236245 (+   390us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:56.236270 (+    25us) write_op.cc:440] Acquired schema lock
1120 03:00:56.236282 (+    12us) tablet.cc:710] Decoding operations
1120 03:00:56.236546 (+   264us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:56.236588 (+    42us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:56.236609 (+    21us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:00:56.236821 (+   212us) tablet.cc:758] Row locks acquired
1120 03:00:56.236835 (+    14us) write_op.cc:260] PREPARE: finished
1120 03:00:56.236948 (+   113us) write_op.cc:270] Start()
1120 03:00:56.237026 (+    78us) write_op.cc:276] Timestamp: P: 1795143656236804 usec, L: 0
1120 03:00:56.237042 (+    16us) op_driver.cc:350] REPLICATION: starting
1120 03:00:56.237514 (+   472us) log.cc:844] Serialized 433 byte log entry
1120 03:00:56.804234 (+566720us) op_driver.cc:466] REPLICATION: finished
1120 03:00:56.826486 (+ 22252us) write_op.cc:301] APPLY: starting
1120 03:00:56.826542 (+    56us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:56.826641 (+    99us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:56.826757 (+   116us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:56.827190 (+   433us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:56.827356 (+   166us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
1120 03:00:56.827400 (+    44us) write_op.cc:312] APPLY: finished
1120 03:00:56.827601 (+   201us) log.cc:844] Serialized 29 byte log entry
1120 03:00:56.828259 (+   658us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:56.828339 (+    80us) write_op.cc:459] Released schema lock
1120 03:00:56.828374 (+    35us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"spinlock_wait_cycles":1191089408,"child_traces":[["op",{"apply.queue_time_us":22081,"num_ops":2,"prepare.queue_time_us":73501,"prepare.run_cpu_time_us":1912,"prepare.run_wall_time_us":138056,"replication_time_us":567112,"threads_started":1}]]}
I20261120 03:00:56.834227   498 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 9882c9cbfbd640a49e3e3c8904ffc1e1 (table default.Table_0879bd445c234e4faefff97cd088035d [id=53fb1849755c4debad2dbfbf37271fbd]). Expected version 1 got 0
W20261120 03:00:56.837875   499 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.65:40295 (request call id 30) took 2847 ms (client timeout 1000 ms). Trace:
W20261120 03:00:56.838968   499 rpcz_store.cc:269] 1120 03:00:53.990464 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.990695 (+   231us) service_pool.cc:224] Handling call
1120 03:00:54.066645 (+ 75950us) spinlock_profiling.cc:245] Waited 74.3 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07d9ce 00007f155abf0174 00007f155abf0102 00007f155abf0031 00007f155abeffac 00007f155abef262 00007f1558e990bc 00007f1558e984f6
1120 03:00:56.837704 (+2771059us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:56.203763 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:56.204219 (+   456us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:56.204253 (+    34us) write_op.cc:440] Acquired schema lock
1120 03:00:56.204265 (+    12us) tablet.cc:710] Decoding operations
1120 03:00:56.211262 (+  6997us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:56.211303 (+    41us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:56.211328 (+    25us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:00:56.211504 (+   176us) tablet.cc:758] Row locks acquired
1120 03:00:56.211517 (+    13us) write_op.cc:260] PREPARE: finished
1120 03:00:56.211664 (+   147us) write_op.cc:270] Start()
1120 03:00:56.211752 (+    88us) write_op.cc:276] Timestamp: P: 1795143656211521 usec, L: 0
1120 03:00:56.211767 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:00:56.212293 (+   526us) log.cc:844] Serialized 527 byte log entry
1120 03:00:56.804000 (+591707us) op_driver.cc:466] REPLICATION: finished
1120 03:00:56.807414 (+  3414us) write_op.cc:301] APPLY: starting
1120 03:00:56.807469 (+    55us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:56.807557 (+    88us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:56.807648 (+    91us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:56.807796 (+   148us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:56.807909 (+   113us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:56.807947 (+    38us) write_op.cc:312] APPLY: finished
1120 03:00:56.808118 (+   171us) log.cc:844] Serialized 23 byte log entry
1120 03:00:56.810208 (+  2090us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:56.810266 (+    58us) write_op.cc:459] Released schema lock
1120 03:00:56.810296 (+    30us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"reported_tablets":6,"spinlock_wait_cycles":170942848,"child_traces":[["op",{"apply.queue_time_us":3286,"num_ops":1,"prepare.queue_time_us":827528,"prepare.run_cpu_time_us":2161,"prepare.run_wall_time_us":32864,"replication_time_us":592163}]]}
W20261120 03:00:56.846021   500 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.67:53803 (request call id 16) took 2505 ms (client timeout 1000 ms). Trace:
W20261120 03:00:56.846513   500 rpcz_store.cc:269] 1120 03:00:54.341025 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.341443 (+   418us) service_pool.cc:224] Handling call
1120 03:00:54.478572 (+137129us) spinlock_profiling.cc:245] Waited 116 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07d9ce 00007f155abf0174 00007f155abf0102 00007f155abf0031 00007f155abeffac 00007f155abef262 00007f1558e990bc 00007f1558e984f6
1120 03:00:56.845948 (+2367376us) inbound_call.cc:177] Queueing success response
Metrics: {"reported_tablets":6,"spinlock_wait_cycles":267481984}
I20261120 03:00:56.803889  1324 ts_tablet_manager.cc:1480] T 014bd9c83be34c31b90fe4bcf31f1649 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.032s	user 0.006s	sys 0.001s
I20261120 03:00:56.809752  1610 tablet_replica.cc:333] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:56.838444   752 tablet_service.cc:1542] Processing DeleteTablet for tablet 014bd9c83be34c31b90fe4bcf31f1649 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:54 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:56.839411   753 tablet_service.cc:1542] Processing DeleteTablet for tablet ce612f9a121b4b18908724d2726e156f with delete_type TABLET_DATA_DELETED (replaced by tablet 5da3d2724dd34aed88e6b08983188004) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:56.816923   710 leader_election.cc:304] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:56.857908  1461 raft_consensus.cc:3024] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:56.858323  1461 raft_consensus.cc:534] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:56.858461   710 leader_election.cc:304] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:56.858632  1461 raft_consensus.cc:3296] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:56.859165  1554 consensus_queue.cc:1049] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20261120 03:00:56.859449   710 leader_election.cc:343] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:56.859589  1485 raft_consensus.cc:3024] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:56.859828   710 leader_election.cc:304] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:56.859458  1461 raft_consensus.cc:556] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.859936  1485 raft_consensus.cc:534] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:56.861200  1485 raft_consensus.cc:3296] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:56.861917  1485 raft_consensus.cc:556] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.860846  1554 raft_consensus.cc:2966] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:56.827653  1295 ts_tablet_manager.cc:1962] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:56.873057  1324 tablet_bootstrap.cc:492] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:56.875437  1610 raft_consensus.cc:2440] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:56.875948  1610 raft_consensus.cc:2469] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:56.838753   899 tablet_service.cc:1495] Processing CreateTablet for tablet 9dadc05b726346309d6b350f9abb4abb (DEFAULT_TABLE table=default.Table_027fdfc1742a404ea34dfdc00e41ff5a [id=7ec945bb5372463b954d8e5d67879627]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.876983   899 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9dadc05b726346309d6b350f9abb4abb. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.877934  1610 ts_tablet_manager.cc:1962] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:56.840230   901 tablet_service.cc:1495] Processing CreateTablet for tablet b549a06d15db449b851fe9549a963416 (DEFAULT_TABLE table=default.Table_61cb1e03b7e442a8822e32a2295866f3 [id=b5b565c2c2224675baada2ae9b42be54]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:56.882786   901 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet b549a06d15db449b851fe9549a963416. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:56.851992   902 tablet_service.cc:1542] Processing DeleteTablet for tablet 68179488b9bc4e03b65a9f1d434c95b5 with delete_type TABLET_DATA_DELETED (replaced by tablet 4d79fc4c8f1c4a5f83c0dc94337c857e) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:56.852577   898 tablet_service.cc:1542] Processing DeleteTablet for tablet a6ae8007455d4be3b50672b9c6b0fd8f with delete_type TABLET_DATA_DELETED (replaced by tablet 8f1fda857e6c400e80d3d177a3d748ff) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:56.853173   900 tablet_service.cc:1542] Processing DeleteTablet for tablet 014bd9c83be34c31b90fe4bcf31f1649 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:54 UTC) from {username='slave'} at 127.0.0.1:44896
W20261120 03:00:56.895004   488 catalog_manager.cc:4863] Async tablet task AlterTable RPC for tablet 5da3d2724dd34aed88e6b08983188004 (table default.Table_d2bd3060da1545548f422436c9c7c22e [id=f5d860e17cd74d6aa7b08fbbb0f8c75a], current schema version=2) failed: Not found: failed to reset TS proxy: No leader found: Tablet 5da3d2724dd34aed88e6b08983188004 has no consensus state
I20261120 03:00:56.901374   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 06c3bac919e342b09dd6af13aebc387b (table default.Table_91d5519024c0442b9cbce1cf67444e6e [id=927890104a3a4819878b6ae3f13c0e71]) successfully deleted
I20261120 03:00:56.904253  1012 leader_election.cc:304] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:56.905423  1517 raft_consensus.cc:2966] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20261120 03:00:56.906356  1324 tablet_bootstrap.cc:654] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:53.187470  1429 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:00:56.933265  1430 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 3799 milliseconds
I20261120 03:00:56.933455  1300 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
W20261120 03:00:53.161011  1431 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:00:56.939303  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b6f0ff2246cd4772af47812b699e93e8" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:56.940423  1300 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20261120 03:00:56.947813  1392 tablet_replica.cc:333] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:56.948724  1392 raft_consensus.cc:2440] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:56.954159  1300 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20261120 03:00:56.955768  1300 hybrid_clock.cc:648] HybridClock initialized: now 1795143656955675 us; error 97 us; skew 500 ppm
I20261120 03:00:56.837343  1522 raft_consensus.cc:556] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.957031  1300 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20261120 03:00:56.965663  1324 tablet_bootstrap.cc:492] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:56.966112  1324 ts_tablet_manager.cc:1449] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.093s	user 0.004s	sys 0.006s
I20261120 03:00:56.827052   857 leader_election.cc:304] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:56.971890  1012 leader_election.cc:304] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:56.974598  1567 raft_consensus.cc:3024] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:56.974920   710 leader_election.cc:304] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:56.975463  1567 raft_consensus.cc:744] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Becoming Leader. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Running, Role: LEADER
I20261120 03:00:56.976361  1567 consensus_queue.cc:238] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.982867  1437 raft_consensus.cc:3024] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:56.983367  1437 raft_consensus.cc:744] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:00:56.984143  1437 consensus_queue.cc:238] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.986492  1461 leader_election.cc:290] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:56.987670  1324 raft_consensus.cc:366] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.988284  1324 raft_consensus.cc:392] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:56.988531  1324 raft_consensus.cc:787] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:56.989115  1324 consensus_queue.cc:261] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:56.991379  1324 ts_tablet_manager.cc:1480] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.025s	user 0.007s	sys 0.001s
I20261120 03:00:56.992249  1324 tablet_bootstrap.cc:492] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:56.995885  1496 raft_consensus.cc:3024] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:56.996440  1496 raft_consensus.cc:744] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:56.997298  1496 consensus_queue.cc:238] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.002246   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5c5c0ac869a1419fad9f61ec647027ee" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:57.002962   923 raft_consensus.cc:3296] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:57.003873   923 raft_consensus.cc:2062] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:57.010926  1324 tablet_bootstrap.cc:654] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:57.012462  1461 consensus_queue.cc:1049] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20261120 03:00:57.014209   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 187225us
I20261120 03:00:57.020969   499 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_b2cdcfa34e9949afa95603981f35136f"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:57.022737   499 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_b2cdcfa34e9949afa95603981f35136f in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:56.978098  1392 raft_consensus.cc:2469] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:57.037133   500 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_8f246cc5b3c24316a8c07be0aff44a59" } new_table_name: "default.Table_47101d55d8f6485c816a02a54fda67a8" modify_external_catalogs: true
I20261120 03:00:57.038429  1392 ts_tablet_manager.cc:1962] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:57.040292   496 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_7b88ce43c2c84af2910d96587817965b"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:57.042742   493 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_faa63585fa0a40f7b9fe1e97441cdc7a" } new_table_name: "default.Table_9fe8b8c914534e4ca5602de7c1b1d280" modify_external_catalogs: true
I20261120 03:00:57.043468  1522 leader_election.cc:290] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:57.044847   918 raft_consensus.cc:2062] T 168cb98bb1b242a5a0377bf292204d96 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
W20261120 03:00:57.041853   496 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_7b88ce43c2c84af2910d96587817965b in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:56.981115  1181 tablet_bootstrap.cc:654] T 68179488b9bc4e03b65a9f1d434c95b5 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:57.018378  1600 log.cc:927] Time spent T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234: Append to log took a long time: real 0.061s	user 0.001s	sys 0.000s
I20261120 03:00:56.973219  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "c6fea104598e4f6d876f96675f2da816" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:57.071750   710 leader_election.cc:304] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:57.073712  1597 raft_consensus.cc:3024] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:57.074203  1597 raft_consensus.cc:744] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:57.074782  1180 tablet_bootstrap.cc:700] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440: Time spent opening tablet: real 0.249s	user 0.004s	sys 0.000s
I20261120 03:00:57.075589  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5c5c0ac869a1419fad9f61ec647027ee" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:57.075515  1597 consensus_queue.cc:238] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.088301  1180 tablet_bootstrap.cc:654] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:57.090569  1485 leader_election.cc:290] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
W20261120 03:00:57.090431   857 consensus_peers.cc:597] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 -> Peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Couldn't send request to peer 5beeefe8bef54caeac463d05d8c30fa6. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: STOPPING. This is attempt 1: this message will repeat every 5th retry.
W20261120 03:00:57.100157   709 leader_election.cc:343] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:57.101831   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a08aa5ddcf1a4061a70275a8eecb1856" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:57.102427   923 raft_consensus.cc:3296] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
W20261120 03:00:57.106356  1078 thread.cc:642] wal-append [worker] (thread pool) Time spent creating pthread: real 0.903s	user 0.013s	sys 0.009s
I20261120 03:00:57.106524  1500 consensus_queue.cc:1049] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20261120 03:00:57.107203  1078 thread.cc:609] wal-append [worker] (thread pool) Time spent starting thread: real 0.904s	user 0.014s	sys 0.009s
I20261120 03:00:57.110359  1500 consensus_queue.cc:1049] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:57.113412   923 raft_consensus.cc:2062] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:00:57.114524  1070 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "35f0ff39b36148e8abfdea3435a3b8b4" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:57.120769  1078 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.96.65:39215 (request call id 95) took 1157 ms. Trace:
I20261120 03:00:57.135906  1181 tablet_bootstrap.cc:492] T 68179488b9bc4e03b65a9f1d434c95b5 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:57.128400  1567 consensus_queue.cc:1049] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:57.136523  1181 ts_tablet_manager.cc:1449] T 68179488b9bc4e03b65a9f1d434c95b5 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.428s	user 0.009s	sys 0.002s
I20261120 03:00:57.139271  1300 webserver.cc:577] Webserver started at http://127.0.96.124:43247/ using document root <none> and password file <none>
I20261120 03:00:57.140570  1300 fs_manager.cc:362] Metadata directory not provided
I20261120 03:00:57.140852  1300 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20261120 03:00:57.141103  1381 raft_consensus.cc:534] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 3d30c2428730481db9b0ffd279dd3440)
I20261120 03:00:57.141659  1381 raft_consensus.cc:556] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:57.144271  1381 leader_election.cc:290] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
W20261120 03:00:57.147037  1012 leader_election.cc:343] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Not found: Tablet not found: 9a31431495274c648625b75900b4ac47
I20261120 03:00:57.149618   709 leader_election.cc:304] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
W20261120 03:00:57.150466   709 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 50341.9us
I20261120 03:00:57.152354  1485 raft_consensus.cc:2966] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20261120 03:00:57.103611   710 leader_election.cc:304] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:57.107815  1324 tablet_bootstrap.cc:492] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:57.154235  1324 ts_tablet_manager.cc:1449] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.162s	user 0.011s	sys 0.000s
W20261120 03:00:57.154201   710 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 50676.4us
I20261120 03:00:57.155246  1485 raft_consensus.cc:3024] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:57.155685  1485 raft_consensus.cc:744] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:57.156476  1485 consensus_queue.cc:238] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.156682  1324 raft_consensus.cc:366] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.157254  1324 raft_consensus.cc:392] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:57.157487  1324 raft_consensus.cc:787] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:57.158016  1324 consensus_queue.cc:261] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.160241  1324 ts_tablet_manager.cc:1480] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.000s
I20261120 03:00:57.161087  1324 tablet_bootstrap.cc:492] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
W20261120 03:00:57.161135   857 consensus_peers.cc:597] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 1: this message will repeat every 5th retry.
W20261120 03:00:57.161746   857 consensus_peers.cc:597] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
W20261120 03:00:57.162402   857 leader_election.cc:343] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:57.163163  1522 consensus_queue.cc:1049] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:57.164942  1300 fs_manager.cc:714] Time spent opening directory manager: real 0.020s	user 0.003s	sys 0.005s
I20261120 03:00:57.166505  1324 tablet_bootstrap.cc:654] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:57.167456  1051 tablet_service.cc:1542] Processing DeleteTablet for tablet 68179488b9bc4e03b65a9f1d434c95b5 with delete_type TABLET_DATA_DELETED (replaced by tablet 4d79fc4c8f1c4a5f83c0dc94337c857e) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:57.169008   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "35f0ff39b36148e8abfdea3435a3b8b4" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:57.169772   776 raft_consensus.cc:2062] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:57.119992  1595 raft_consensus.cc:534] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:57.126048  1610 ts_tablet_manager.cc:1975] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:57.170367  1051 tablet_service.cc:1542] Processing DeleteTablet for tablet a6ae8007455d4be3b50672b9c6b0fd8f with delete_type TABLET_DATA_DELETED (replaced by tablet 8f1fda857e6c400e80d3d177a3d748ff) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:57.170730  1610 log.cc:1199] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/68179488b9bc4e03b65a9f1d434c95b5
I20261120 03:00:57.171341  1051 tablet_service.cc:1542] Processing DeleteTablet for tablet 014bd9c83be34c31b90fe4bcf31f1649 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:54 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:57.175302  1610 ts_tablet_manager.cc:1996] T 68179488b9bc4e03b65a9f1d434c95b5 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:57.176182  1051 tablet_service.cc:1542] Processing DeleteTablet for tablet fa57d0c282964e37b9133ea57f458a51 with delete_type TABLET_DATA_DELETED (replaced by tablet c6a65c1a4a574961bec657ad7c9048e0) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:57.176803  1610 tablet_replica.cc:333] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:57.176976   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 68179488b9bc4e03b65a9f1d434c95b5 (table default.Table_650dc67e0ac74c149a3f64d456767938 [id=cfcd5fea97fb4ab3baff99eb28cd9476]) successfully deleted
W20261120 03:00:57.177228  1011 consensus_peers.cc:597] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 -> Peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Couldn't send request to peer 5beeefe8bef54caeac463d05d8c30fa6. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ce612f9a121b4b18908724d2726e156f. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:57.177627  1610 raft_consensus.cc:2440] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:57.180902  1595 raft_consensus.cc:556] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.181809  1181 raft_consensus.cc:366] T 68179488b9bc4e03b65a9f1d434c95b5 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:57.178221  1610 raft_consensus.cc:2469] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:57.185103  1635 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:00:57.185034  1595 leader_election.cc:290] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:57.186671  1300 fs_manager.cc:730] Time spent opening block manager: real 0.018s	user 0.005s	sys 0.002s
I20261120 03:00:57.188795  1180 tablet_bootstrap.cc:492] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:57.189317  1180 ts_tablet_manager.cc:1449] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.658s	user 0.010s	sys 0.001s
I20261120 03:00:57.187103  1300 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/data,/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/wal
uuid: "2599fd9eef7d4ce990c40e2356a37c3f"
format_stamp: "Formatted at 2026-11-20 03:00:38 on dist-test-slave-jpwh"
I20261120 03:00:57.189924  1300 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/wal
metadata directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/wal
1 data directories: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20261120 03:00:57.192538  1180 raft_consensus.cc:366] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.193248  1180 raft_consensus.cc:392] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
W20261120 03:00:57.201359   498 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.66:36141 (request call id 24) took 3104 ms (client timeout 1000 ms). Trace:
W20261120 03:00:57.202005   498 rpcz_store.cc:269] 1120 03:00:54.096700 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.102182 (+  5482us) service_pool.cc:224] Handling call
1120 03:00:57.201288 (+3099106us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:56.837001 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:56.837443 (+   442us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:56.837471 (+    28us) write_op.cc:440] Acquired schema lock
1120 03:00:56.837483 (+    12us) tablet.cc:710] Decoding operations
1120 03:00:56.837773 (+   290us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:56.837813 (+    40us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:56.837836 (+    23us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:00:56.837997 (+   161us) tablet.cc:758] Row locks acquired
1120 03:00:56.838010 (+    13us) write_op.cc:260] PREPARE: finished
1120 03:00:56.838159 (+   149us) write_op.cc:270] Start()
1120 03:00:56.838244 (+    85us) write_op.cc:276] Timestamp: P: 1795143656838103 usec, L: 0
1120 03:00:56.838259 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:00:57.036571 (+198312us) log.cc:844] Serialized 527 byte log entry
1120 03:00:57.045193 (+  8622us) spinlock_profiling.cc:245] Waited 198 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b147a41 00007f155b1db018 00007f155b68b381 00007f155b68ab58 00007f155b68f7f1 00007f155b68f7a9 00007f155b68f739 00007f155b68f701 00007f155b68e92d 0000000000327654 0000000000327479
1120 03:00:57.192147 (+146954us) op_driver.cc:466] REPLICATION: finished
1120 03:00:57.192432 (+   285us) write_op.cc:301] APPLY: starting
1120 03:00:57.192501 (+    69us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:57.192593 (+    92us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:57.192612 (+    19us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:57.192801 (+   189us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:57.192932 (+   131us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:57.192987 (+    55us) write_op.cc:312] APPLY: finished
1120 03:00:57.193194 (+   207us) log.cc:844] Serialized 23 byte log entry
1120 03:00:57.193353 (+   159us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:57.193418 (+    65us) write_op.cc:459] Released schema lock
1120 03:00:57.193448 (+    30us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"mutex_wait_us":31196,"reported_tablets":4,"child_traces":[["op",{"apply.queue_time_us":104,"apply.run_cpu_time_us":1234,"apply.run_wall_time_us":1234,"num_ops":1,"prepare.queue_time_us":105,"prepare.run_cpu_time_us":13996,"prepare.run_wall_time_us":352992,"raft.queue_time_us":1158,"raft.run_cpu_time_us":799,"raft.run_wall_time_us":799,"replication_time_us":353788,"spinlock_wait_cycles":454848384,"thread_start_us":144490,"threads_started":1}]]}
I20261120 03:00:57.206308  1555 consensus_queue.cc:1049] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:57.210841  1610 ts_tablet_manager.cc:1962] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:57.219853  1324 tablet_bootstrap.cc:492] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:57.220284  1324 ts_tablet_manager.cc:1449] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.059s	user 0.007s	sys 0.002s
W20261120 03:00:57.224373   710 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 45085.5us
W20261120 03:00:57.225538  1631 log.cc:927] Time spent T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440: Append to log took a long time: real 0.050s	user 0.000s	sys 0.000s
I20261120 03:00:57.225921   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b3f7103b59d443f9bc739083b8e023ce" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:57.235137   857 leader_election.cc:304] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:57.239307  1324 raft_consensus.cc:366] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.239984  1324 raft_consensus.cc:392] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:57.240232  1324 raft_consensus.cc:787] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:57.240818  1324 consensus_queue.cc:261] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.242775  1324 ts_tablet_manager.cc:1480] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.022s	user 0.007s	sys 0.000s
I20261120 03:00:57.243614  1324 tablet_bootstrap.cc:492] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:57.243970  1181 raft_consensus.cc:392] T 68179488b9bc4e03b65a9f1d434c95b5 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:57.135970  1078 rpcz_store.cc:276] 1120 03:00:55.963098 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:55.963209 (+   111us) service_pool.cc:224] Handling call
1120 03:00:55.963543 (+   334us) raft_consensus.cc:1536] Updating replica for 1 ops
1120 03:00:56.085422 (+121879us) raft_consensus.cc:1602] Early marking committed up to index 0
1120 03:00:56.088673 (+  3251us) raft_consensus.cc:1607] Triggering prepare for 1 ops
1120 03:00:56.202437 (+113764us) log.cc:844] Serialized 26 byte log entry
1120 03:00:57.108468 (+906031us) raft_consensus.cc:1713] Marking committed up to 0
1120 03:00:57.108510 (+    42us) raft_consensus.cc:1765] Filling consensus response to leader.
1120 03:00:57.120516 (+ 12006us) raft_consensus.cc:1736] Waiting on the replicates to finish logging
1120 03:00:57.120567 (+    51us) raft_consensus.cc:1753] finished
1120 03:00:57.120578 (+    11us) raft_consensus.cc:1759] UpdateReplicas() finished
1120 03:00:57.120712 (+   134us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":21978,"raft.queue_time_us":32908,"raft.run_cpu_time_us":123,"raft.run_wall_time_us":123,"thread_start_us":984969,"threads_started":2,"wal-append.queue_time_us":753836}
W20261120 03:00:57.222157  1011 leader_election.cc:343] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: 9a31431495274c648625b75900b4ac47
I20261120 03:00:57.222863  1070 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a08aa5ddcf1a4061a70275a8eecb1856" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:57.246474  1011 leader_election.cc:304] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:57.247133  1549 raft_consensus.cc:2966] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20261120 03:00:57.245292  1181 raft_consensus.cc:787] T 68179488b9bc4e03b65a9f1d434c95b5 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:57.250859  1181 consensus_queue.cc:261] T 68179488b9bc4e03b65a9f1d434c95b5 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:57.257184  1522 raft_consensus.cc:3024] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:57.257629  1522 raft_consensus.cc:534] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:57.258011  1522 raft_consensus.cc:3296] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:57.259476  1522 raft_consensus.cc:556] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.262092  1549 raft_consensus.cc:534] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:57.262576  1549 raft_consensus.cc:556] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.264489  1549 leader_election.cc:290] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
W20261120 03:00:57.249239   710 leader_election.cc:343] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:57.266678  1012 leader_election.cc:343] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Not found: Tablet not found: 1885397e42774a89bb347bc776284bc0
I20261120 03:00:57.266918  1461 consensus_queue.cc:1049] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20261120 03:00:57.268002   486 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.126:44467, user_credentials={real_user=slave}} blocked reactor thread for 51154.5us
I20261120 03:00:57.272499  1181 ts_tablet_manager.cc:1480] T 68179488b9bc4e03b65a9f1d434c95b5 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.136s	user 0.005s	sys 0.000s
I20261120 03:00:57.193536  1180 raft_consensus.cc:787] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:57.280237  1180 consensus_queue.cc:261] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:57.285619   709 leader_election.cc:343] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:57.288997  1324 tablet_bootstrap.cc:654] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:57.295913   709 leader_election.cc:343] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:57.296902   709 consensus_peers.cc:597] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:57.304608  1324 tablet_bootstrap.cc:492] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:57.305039  1324 ts_tablet_manager.cc:1449] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.062s	user 0.000s	sys 0.008s
I20261120 03:00:57.307696  1324 raft_consensus.cc:366] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.308357   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1885397e42774a89bb347bc776284bc0" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:57.311190  1181 tablet_bootstrap.cc:492] T 35f0ff39b36148e8abfdea3435a3b8b4 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:57.308367  1324 raft_consensus.cc:392] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:57.316082  1324 raft_consensus.cc:787] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:57.311853  1011 leader_election.cc:304] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:57.316754  1549 raft_consensus.cc:2966] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:57.316716  1324 consensus_queue.cc:261] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.323637  1324 ts_tablet_manager.cc:1480] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.018s	user 0.007s	sys 0.001s
W20261120 03:00:57.324196   709 leader_election.cc:343] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:57.324595  1324 tablet_bootstrap.cc:492] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:57.325647  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b3f7103b59d443f9bc739083b8e023ce" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:57.327560  1610 ts_tablet_manager.cc:1975] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:57.327961  1610 log.cc:1199] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/a6ae8007455d4be3b50672b9c6b0fd8f
I20261120 03:00:57.328660  1610 ts_tablet_manager.cc:1996] T a6ae8007455d4be3b50672b9c6b0fd8f P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:57.335537  1610 tablet_replica.cc:333] T 014bd9c83be34c31b90fe4bcf31f1649 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:57.336338  1610 raft_consensus.cc:2440] T 014bd9c83be34c31b90fe4bcf31f1649 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:57.336889  1610 raft_consensus.cc:2469] T 014bd9c83be34c31b90fe4bcf31f1649 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:57.338728  1610 ts_tablet_manager.cc:1962] T 014bd9c83be34c31b90fe4bcf31f1649 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:57.346725   710 leader_election.cc:304] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:57.347817  1595 raft_consensus.cc:3024] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:57.348480  1595 raft_consensus.cc:744] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:57.349241  1595 consensus_queue.cc:238] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.350427  1300 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20261120 03:00:57.352433  1300 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20261120 03:00:57.353653  1300 kserver.cc:163] Server-wide thread pool size limit: 3276
I20261120 03:00:57.358385  1324 tablet_bootstrap.cc:654] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:57.359205  1295 ts_tablet_manager.cc:1975] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:57.359639  1295 log.cc:1199] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/020be7f7cda0467aab56f657cbe8d0ad
I20261120 03:00:57.360386  1295 ts_tablet_manager.cc:1996] T 020be7f7cda0467aab56f657cbe8d0ad P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:57.378296  1181 tablet_bootstrap.cc:654] T 35f0ff39b36148e8abfdea3435a3b8b4 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:57.383966   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 82863.3us
I20261120 03:00:57.385245  1180 ts_tablet_manager.cc:1480] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.196s	user 0.005s	sys 0.001s
I20261120 03:00:57.391187  1324 tablet_bootstrap.cc:492] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:57.391564  1324 ts_tablet_manager.cc:1449] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.067s	user 0.010s	sys 0.002s
I20261120 03:00:57.393874  1324 raft_consensus.cc:366] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:57.394501  1324 raft_consensus.cc:392] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:57.394748  1324 raft_consensus.cc:787] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:57.395310  1324 consensus_queue.cc:261] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:57.397213  1324 ts_tablet_manager.cc:1480] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.005s	user 0.001s	sys 0.003s
I20261120 03:00:57.398026  1324 tablet_bootstrap.cc:492] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:57.428609  1324 tablet_bootstrap.cc:654] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:57.428758   500 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 1faba22e8e38418383b86e42e904a761 has no consensus state
I20261120 03:00:57.429220   500 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:45640 (request call id 157) took 1855 ms. Trace:
I20261120 03:00:57.429569   500 rpcz_store.cc:276] 1120 03:00:55.573550 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:56.847969 (+1274419us) service_pool.cc:224] Handling call
1120 03:00:57.037032 (+189063us) spinlock_profiling.cc:245] Waited 188 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07f626 00007f155ac007f4 00007f155ac00782 00007f155ac006b1 00007f155ac0062c 00007f155abff8e2 00007f1558e990bc 00007f1558e984f6
1120 03:00:57.037825 (+   793us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:00:57.039082 (+  1257us) catalog_manager.cc:3769] Apply alter schema
1120 03:00:57.039732 (+   650us) catalog_manager.cc:3801] Acquired catalog manager lock
1120 03:00:57.039997 (+   265us) catalog_manager.cc:3972] Updating metadata on disk
1120 03:00:57.428222 (+388225us) catalog_manager.cc:4003] Committing alterations to in-memory state
1120 03:00:57.429155 (+   933us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:57.198839 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:57.199273 (+   434us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:57.199300 (+    27us) write_op.cc:440] Acquired schema lock
1120 03:00:57.199314 (+    14us) tablet.cc:710] Decoding operations
1120 03:00:57.199600 (+   286us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:57.199644 (+    44us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:57.199666 (+    22us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:00:57.199851 (+   185us) tablet.cc:758] Row locks acquired
1120 03:00:57.199864 (+    13us) write_op.cc:260] PREPARE: finished
1120 03:00:57.199989 (+   125us) write_op.cc:270] Start()
1120 03:00:57.200095 (+   106us) write_op.cc:276] Timestamp: P: 1795143657199936 usec, L: 0
1120 03:00:57.200112 (+    17us) op_driver.cc:350] REPLICATION: starting
1120 03:00:57.200634 (+   522us) log.cc:844] Serialized 405 byte log entry
1120 03:00:57.417679 (+217045us) op_driver.cc:466] REPLICATION: finished
1120 03:00:57.418409 (+   730us) write_op.cc:301] APPLY: starting
1120 03:00:57.418465 (+    56us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:57.418613 (+   148us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:57.418714 (+   101us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:57.419045 (+   331us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:57.419169 (+   124us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:57.419212 (+    43us) write_op.cc:312] APPLY: finished
1120 03:00:57.425232 (+  6020us) log.cc:844] Serialized 23 byte log entry
1120 03:00:57.425542 (+   310us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:57.425608 (+    66us) write_op.cc:459] Released schema lock
1120 03:00:57.425639 (+    31us) write_op.cc:346] FINISH: Updating metrics
1120 03:00:57.425859 (+   220us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:57.425883 (+    24us) write_op.cc:459] Released schema lock
Metrics: {"spinlock_wait_cycles":431725440,"child_traces":[["op",{"apply.queue_time_us":567,"apply.run_cpu_time_us":1619,"apply.run_wall_time_us":7423,"num_ops":1,"prepare.queue_time_us":148926,"prepare.run_cpu_time_us":2155,"prepare.run_wall_time_us":11741,"raft.queue_time_us":4404,"raft.run_cpu_time_us":908,"raft.run_wall_time_us":51836,"replication_time_us":217448}]]}
I20261120 03:00:57.435349   499 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:45640 (request call id 156) took 1889 ms. Trace:
I20261120 03:00:57.435694   499 rpcz_store.cc:276] 1120 03:00:55.545762 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:56.841884 (+1296122us) service_pool.cc:224] Handling call
1120 03:00:57.020866 (+178982us) spinlock_profiling.cc:245] Waited 178 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07efa6 00007f155abf71f4 00007f155abf7182 00007f155abf70b1 00007f155abf702c 00007f155abf62e2 00007f1558e990bc 00007f1558e984f6
1120 03:00:57.023157 (+  2291us) catalog_manager.cc:2145] Acquired catalog manager lock
1120 03:00:57.059559 (+ 36402us) catalog_manager.cc:2191] Created new table and tablet info
1120 03:00:57.434735 (+375176us) catalog_manager.cc:2251] Wrote table and tablets to system table
1120 03:00:57.435029 (+   294us) catalog_manager.cc:2285] Inserted table and tablets into CatalogManager maps
1120 03:00:57.435292 (+   263us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:57.201999 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:57.202463 (+   464us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:57.202487 (+    24us) write_op.cc:440] Acquired schema lock
1120 03:00:57.202501 (+    14us) tablet.cc:710] Decoding operations
1120 03:00:57.202779 (+   278us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:57.202817 (+    38us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:57.202839 (+    22us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:00:57.203056 (+   217us) tablet.cc:758] Row locks acquired
1120 03:00:57.203073 (+    17us) write_op.cc:260] PREPARE: finished
1120 03:00:57.203181 (+   108us) write_op.cc:270] Start()
1120 03:00:57.203271 (+    90us) write_op.cc:276] Timestamp: P: 1795143657203132 usec, L: 0
1120 03:00:57.203288 (+    17us) op_driver.cc:350] REPLICATION: starting
1120 03:00:57.203771 (+   483us) log.cc:844] Serialized 433 byte log entry
1120 03:00:57.417927 (+214156us) op_driver.cc:466] REPLICATION: finished
1120 03:00:57.426124 (+  8197us) write_op.cc:301] APPLY: starting
1120 03:00:57.426173 (+    49us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:57.426274 (+   101us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:57.426363 (+    89us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:57.426774 (+   411us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:57.426909 (+   135us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
1120 03:00:57.426947 (+    38us) write_op.cc:312] APPLY: finished
1120 03:00:57.427121 (+   174us) log.cc:844] Serialized 29 byte log entry
1120 03:00:57.427240 (+   119us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:57.427295 (+    55us) write_op.cc:459] Released schema lock
1120 03:00:57.427326 (+    31us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"spinlock_wait_cycles":409826176,"child_traces":[["op",{"apply.queue_time_us":8056,"num_ops":2,"prepare.queue_time_us":141130,"prepare.run_cpu_time_us":2061,"prepare.run_wall_time_us":102200,"replication_time_us":214557,"threads_started":1}]]}
I20261120 03:00:57.443081  1078 raft_consensus.cc:3296] T 68179488b9bc4e03b65a9f1d434c95b5 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:57.450795   499 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_dd2f7617ff0e401b9b862d5991110dd5" } modify_external_catalogs: true
I20261120 03:00:57.451393   499 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_dd2f7617ff0e401b9b862d5991110dd5" } modify_external_catalogs: true
I20261120 03:00:57.454239   500 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_331ec71dbe224231bb5e1cce1470d2ea"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:57.455735   500 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_331ec71dbe224231bb5e1cce1470d2ea in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:57.468551   774 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "35f0ff39b36148e8abfdea3435a3b8b4" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:57.469146   774 raft_consensus.cc:3296] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:57.470088   774 raft_consensus.cc:2062] T 35f0ff39b36148e8abfdea3435a3b8b4 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:00:57.481928  1070 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "35f0ff39b36148e8abfdea3435a3b8b4" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:57.493688  1485 consensus_queue.cc:1049] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:57.503422  1181 tablet_bootstrap.cc:492] T 35f0ff39b36148e8abfdea3435a3b8b4 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:57.503865  1181 ts_tablet_manager.cc:1449] T 35f0ff39b36148e8abfdea3435a3b8b4 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.195s	user 0.009s	sys 0.005s
I20261120 03:00:57.509948  1324 tablet_bootstrap.cc:492] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:57.510414  1324 ts_tablet_manager.cc:1449] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.113s	user 0.018s	sys 0.004s
I20261120 03:00:57.512255  1610 ts_tablet_manager.cc:1975] T 014bd9c83be34c31b90fe4bcf31f1649 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:57.512629  1610 log.cc:1199] T 014bd9c83be34c31b90fe4bcf31f1649 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/014bd9c83be34c31b90fe4bcf31f1649
I20261120 03:00:57.513224  1610 ts_tablet_manager.cc:1996] T 014bd9c83be34c31b90fe4bcf31f1649 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
W20261120 03:00:57.519378   486 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.124:38133, user_credentials={real_user=slave}} blocked reactor thread for 105615us
I20261120 03:00:57.523779  1181 raft_consensus.cc:366] T 35f0ff39b36148e8abfdea3435a3b8b4 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.524430  1181 raft_consensus.cc:392] T 35f0ff39b36148e8abfdea3435a3b8b4 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:57.524673  1181 raft_consensus.cc:787] T 35f0ff39b36148e8abfdea3435a3b8b4 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:57.526062  1324 raft_consensus.cc:366] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.526697  1324 raft_consensus.cc:392] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:57.526932  1324 raft_consensus.cc:787] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:57.527465  1324 consensus_queue.cc:261] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.448433  1522 leader_election.cc:290] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
W20261120 03:00:57.483731   857 leader_election.cc:343] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:57.533463   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 49772.5us
I20261120 03:00:57.533313  1181 consensus_queue.cc:261] T 35f0ff39b36148e8abfdea3435a3b8b4 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.536490   857 leader_election.cc:304] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:57.537284  1522 raft_consensus.cc:3024] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:57.537715  1522 raft_consensus.cc:744] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:00:57.538523  1522 consensus_queue.cc:238] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.539388  1324 ts_tablet_manager.cc:1480] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.029s	user 0.006s	sys 0.000s
I20261120 03:00:57.543299  1597 raft_consensus.cc:534] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:57.543838  1597 raft_consensus.cc:556] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.545868  1181 ts_tablet_manager.cc:1480] T 35f0ff39b36148e8abfdea3435a3b8b4 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.042s	user 0.004s	sys 0.004s
I20261120 03:00:57.548009  1324 tablet_bootstrap.cc:492] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:57.548113   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "f376fdc03b3b4e3680171af6b8574c2e" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:00:57.549521   710 leader_election.cc:343] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:57.555033   710 consensus_peers.cc:597] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Couldn't send request to peer 3d30c2428730481db9b0ffd279dd3440. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 020be7f7cda0467aab56f657cbe8d0ad. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:57.554968  1597 leader_election.cc:290] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:57.551887  1181 tablet_bootstrap.cc:492] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:57.566972  1078 raft_consensus.cc:3296] T 35f0ff39b36148e8abfdea3435a3b8b4 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:57.571277  1612 consensus_queue.cc:1049] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20261120 03:00:57.584985  1324 tablet_bootstrap.cc:654] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:57.587750  1612 consensus_queue.cc:1049] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:57.590379  1180 tablet_bootstrap.cc:492] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:57.604741  1597 raft_consensus.cc:534] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:57.605980  1392 ts_tablet_manager.cc:1975] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:57.605304  1597 raft_consensus.cc:556] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:57.606400  1392 log.cc:1199] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/a642c32c1b6043978516d97efa2347a3
I20261120 03:00:57.609544   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "c6a65c1a4a574961bec657ad7c9048e0" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:57.610812   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet a6ae8007455d4be3b50672b9c6b0fd8f (table default.Table_f73e1375172b4e9999e9906c1491c0d5 [id=96692554644a476d87e6bdcaa3bd847c]) successfully deleted
W20261120 03:00:57.611176   710 leader_election.cc:343] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:57.611398   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 014bd9c83be34c31b90fe4bcf31f1649 (table default.Table_cfa8989fb1134d908ab28bd6ebb80563 [id=16bb3f7e35c54a8c936b4af5a66a7f17]) successfully deleted
W20261120 03:00:57.611994   486 catalog_manager.cc:5048] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet ce612f9a121b4b18908724d2726e156f because the tablet was not found. No further retry: Not found: Tablet not found: ce612f9a121b4b18908724d2726e156f
I20261120 03:00:57.612139  1597 leader_election.cc:290] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:57.613226  1181 tablet_bootstrap.cc:654] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:57.624010  1392 ts_tablet_manager.cc:1996] T a642c32c1b6043978516d97efa2347a3 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:57.629163  1324 tablet_bootstrap.cc:492] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:57.629704  1324 ts_tablet_manager.cc:1449] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.090s	user 0.011s	sys 0.001s
I20261120 03:00:57.635085  1324 raft_consensus.cc:366] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:57.640532  1181 tablet_bootstrap.cc:492] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:57.641019  1181 ts_tablet_manager.cc:1449] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.089s	user 0.012s	sys 0.000s
I20261120 03:00:57.648011  1324 raft_consensus.cc:392] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:57.648399  1324 raft_consensus.cc:787] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:57.643347  1181 raft_consensus.cc:366] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.649637  1324 consensus_queue.cc:261] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:57.507068  1295 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 83) took 3882 ms. Trace:
I20261120 03:00:57.654275  1181 raft_consensus.cc:392] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:57.654217  1295 rpcz_store.cc:276] 1120 03:00:53.624628 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.624768 (+   140us) service_pool.cc:224] Handling call
1120 03:00:56.583611 (+2958843us) ts_tablet_manager.cc:1197] Deleting tablet 020be7f7cda0467aab56f657cbe8d0ad
1120 03:00:56.583656 (+    45us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:57.227722 (+644066us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:57.359074 (+131352us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:57.507005 (+147931us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2956730}
I20261120 03:00:57.654896  1181 raft_consensus.cc:787] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:57.655829   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 020be7f7cda0467aab56f657cbe8d0ad (table default.Table_d1737969bdd644a6ab39da121eeedd60 [id=ed325a58ff124ba9a6b1d05bc901793b]) successfully deleted
W20261120 03:00:57.659639   709 consensus_peers.cc:597] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:57.673851  1461 raft_consensus.cc:534] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:57.680037  1681 tablet.cc:1872] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 3d30c2428730481db9b0ffd279dd3440: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20261120 03:00:57.680318  1461 raft_consensus.cc:556] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.682405  1461 leader_election.cc:290] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:57.683887   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "600a18b7e5104a78935d906b8adfde6f" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:57.685381  1295 tablet_replica.cc:333] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
W20261120 03:00:57.685403   710 leader_election.cc:343] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:57.686147  1295 raft_consensus.cc:2440] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
W20261120 03:00:57.686148   709 leader_election.cc:343] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:57.686606  1295 raft_consensus.cc:2469] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:57.686483   709 leader_election.cc:304] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
W20261120 03:00:57.688052   709 consensus_peers.cc:597] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:57.690150  1295 ts_tablet_manager.cc:1962] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:57.694250  1485 raft_consensus.cc:2966] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:57.710493  1597 raft_consensus.cc:534] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:57.710956  1597 raft_consensus.cc:556] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.714507   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "091dd73d27df4db09ab750747da7584e" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:00:57.715840   710 leader_election.cc:343] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:57.716293  1597 leader_election.cc:290] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
W20261120 03:00:57.683991  1659 log.cc:927] Time spent T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440: Append to log took a long time: real 0.051s	user 0.001s	sys 0.000s
I20261120 03:00:57.720269  1324 ts_tablet_manager.cc:1480] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.088s	user 0.005s	sys 0.000s
I20261120 03:00:57.721101  1324 tablet_bootstrap.cc:492] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:57.700137  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "c6a65c1a4a574961bec657ad7c9048e0" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:57.705163  1068 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "600a18b7e5104a78935d906b8adfde6f" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:57.730545  1181 consensus_queue.cc:261] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.705865  1070 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "f376fdc03b3b4e3680171af6b8574c2e" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:57.732501  1181 ts_tablet_manager.cc:1480] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.091s	user 0.003s	sys 0.000s
I20261120 03:00:57.733330  1181 tablet_bootstrap.cc:492] T 168cb98bb1b242a5a0377bf292204d96 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
W20261120 03:00:57.738646   709 consensus_peers.cc:597] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:57.738529  1070 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "091dd73d27df4db09ab750747da7584e" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:57.711709  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 78) took 3863 ms. Trace:
I20261120 03:00:57.740164  1392 rpcz_store.cc:276] 1120 03:00:53.848131 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.848275 (+   144us) service_pool.cc:224] Handling call
1120 03:00:56.711338 (+2863063us) ts_tablet_manager.cc:1197] Deleting tablet a642c32c1b6043978516d97efa2347a3
1120 03:00:56.947668 (+236330us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:57.348870 (+401202us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:57.605856 (+256986us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:57.711636 (+105780us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2862405}
I20261120 03:00:57.741688  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 79) took 3886 ms. Trace:
I20261120 03:00:57.742084  1392 rpcz_store.cc:276] 1120 03:00:53.855195 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.855366 (+   171us) service_pool.cc:224] Handling call
1120 03:00:57.741364 (+3885998us) ts_tablet_manager.cc:1197] Deleting tablet a72d5d7896de4020a1ac4d2f8c551e15
1120 03:00:57.741426 (+    62us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:57.741635 (+   209us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3885415}
I20261120 03:00:57.743108  1392 tablet_replica.cc:333] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:57.743906  1392 raft_consensus.cc:2440] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:57.751036  1392 raft_consensus.cc:2469] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:57.753227  1392 ts_tablet_manager.cc:1962] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
W20261120 03:00:57.754799   709 leader_election.cc:343] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:57.755294   709 leader_election.cc:304] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:57.756120  1595 raft_consensus.cc:2966] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:57.758904  1180 tablet_bootstrap.cc:654] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:57.766039   709 leader_election.cc:343] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:57.766438   709 leader_election.cc:304] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:57.772459  1595 raft_consensus.cc:2966] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20261120 03:00:57.773511   709 leader_election.cc:343] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:57.773855   709 leader_election.cc:304] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:57.775170  1324 tablet_bootstrap.cc:654] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:57.777479  1595 raft_consensus.cc:2966] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20261120 03:00:57.778975   709 leader_election.cc:343] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:57.779307   709 leader_election.cc:304] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:57.783229  1595 raft_consensus.cc:2966] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:57.800189  1692 tablet.cc:1872] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 5beeefe8bef54caeac463d05d8c30fa6: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20261120 03:00:57.811293   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet a642c32c1b6043978516d97efa2347a3 (table default.Table_e617c39b20b24b8f9477f625da46b661 [id=2b97828a5a874a8cb13e283ef679c7fb]) successfully deleted
I20261120 03:00:57.811642  1324 tablet_bootstrap.cc:492] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:57.812022  1324 ts_tablet_manager.cc:1449] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.091s	user 0.006s	sys 0.002s
W20261120 03:00:57.811954   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet a72d5d7896de4020a1ac4d2f8c551e15 because the tablet was not found. No further retry: Not found: Tablet not found: a72d5d7896de4020a1ac4d2f8c551e15
I20261120 03:00:57.762894  1295 ts_tablet_manager.cc:1975] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:57.812745  1295 log.cc:1199] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/5c5c0ac869a1419fad9f61ec647027ee
I20261120 03:00:57.813442  1295 ts_tablet_manager.cc:1996] T 5c5c0ac869a1419fad9f61ec647027ee P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:57.815476  1295 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 87) took 3728 ms. Trace:
I20261120 03:00:57.815789  1295 rpcz_store.cc:276] 1120 03:00:54.087537 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.095251 (+  7714us) service_pool.cc:224] Handling call
1120 03:00:57.655471 (+3560220us) ts_tablet_manager.cc:1197] Deleting tablet 5c5c0ac869a1419fad9f61ec647027ee
1120 03:00:57.685267 (+ 29796us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:57.708189 (+ 22922us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:57.762779 (+ 54590us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:57.815419 (+ 52640us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3557655}
I20261120 03:00:57.815929  1324 raft_consensus.cc:366] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.816514  1324 raft_consensus.cc:392] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:57.816758  1324 raft_consensus.cc:787] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:57.817003   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 5c5c0ac869a1419fad9f61ec647027ee (table default.Table_5c5911c8746d4b26843ec77c8141d3b6 [id=ffb2040a2e114893855d4d8c66dfa855]) successfully deleted
I20261120 03:00:57.817309  1324 consensus_queue.cc:261] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.817999  1295 tablet_replica.cc:333] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:57.818854  1295 raft_consensus.cc:2440] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:57.821152  1324 ts_tablet_manager.cc:1480] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.009s	user 0.006s	sys 0.001s
I20261120 03:00:57.821857  1295 raft_consensus.cc:2469] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:57.782884  1181 tablet_bootstrap.cc:654] T 168cb98bb1b242a5a0377bf292204d96 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:57.803819  1381 raft_consensus.cc:534] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:57.830022  1381 raft_consensus.cc:556] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:57.831997  1381 leader_election.cc:290] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
W20261120 03:00:57.834545  1012 leader_election.cc:343] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Not found: Tablet not found: df61317717d3401eb74cd93aa336c4dc
W20261120 03:00:57.835132  1011 leader_election.cc:343] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: df61317717d3401eb74cd93aa336c4dc
I20261120 03:00:57.835353  1295 ts_tablet_manager.cc:1962] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:57.835542  1011 leader_election.cc:304] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:57.836284  1381 raft_consensus.cc:2966] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:57.836210  1693 tablet.cc:1872] T 9882c9cbfbd640a49e3e3c8904ffc1e1 P 823abdbd90ba47bbacad0f8dcff78234: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20261120 03:00:57.840070  1181 tablet_bootstrap.cc:492] T 168cb98bb1b242a5a0377bf292204d96 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:57.840466  1181 ts_tablet_manager.cc:1449] T 168cb98bb1b242a5a0377bf292204d96 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.107s	user 0.008s	sys 0.000s
I20261120 03:00:57.842869  1181 raft_consensus.cc:366] T 168cb98bb1b242a5a0377bf292204d96 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.843473  1181 raft_consensus.cc:392] T 168cb98bb1b242a5a0377bf292204d96 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:57.843731  1181 raft_consensus.cc:787] T 168cb98bb1b242a5a0377bf292204d96 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:57.844416  1181 consensus_queue.cc:261] T 168cb98bb1b242a5a0377bf292204d96 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.846634  1181 ts_tablet_manager.cc:1480] T 168cb98bb1b242a5a0377bf292204d96 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.006s	user 0.007s	sys 0.000s
I20261120 03:00:57.847486  1181 tablet_bootstrap.cc:492] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:57.853227  1381 raft_consensus.cc:534] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:57.853660  1381 raft_consensus.cc:556] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.855512  1381 leader_election.cc:290] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:57.855784  1180 tablet_bootstrap.cc:492] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:57.856177  1180 ts_tablet_manager.cc:1449] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.266s	user 0.009s	sys 0.000s
I20261120 03:00:57.857846   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b6f0ff2246cd4772af47812b699e93e8" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:57.858618   776 raft_consensus.cc:2628] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 823abdbd90ba47bbacad0f8dcff78234 in current term 1: Already voted for candidate 5beeefe8bef54caeac463d05d8c30fa6 in this term.
I20261120 03:00:57.861511   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b6f0ff2246cd4772af47812b699e93e8" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:57.862249   918 raft_consensus.cc:2628] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 823abdbd90ba47bbacad0f8dcff78234 in current term 1: Already voted for candidate 3d30c2428730481db9b0ffd279dd3440 in this term.
I20261120 03:00:57.863559  1012 leader_election.cc:304] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:57.863996  1180 raft_consensus.cc:366] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.864609  1180 raft_consensus.cc:392] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:57.864847  1180 raft_consensus.cc:787] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:57.864871  1381 raft_consensus.cc:3296] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:57.865557  1381 raft_consensus.cc:2966] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:57.865546  1180 consensus_queue.cc:261] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.873208  1181 tablet_bootstrap.cc:654] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:57.878181  1295 ts_tablet_manager.cc:1975] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:57.878574  1295 log.cc:1199] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/ce612f9a121b4b18908724d2726e156f
I20261120 03:00:57.882498  1295 ts_tablet_manager.cc:1996] T ce612f9a121b4b18908724d2726e156f P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:57.885118  1295 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 89) took 3541 ms. Trace:
I20261120 03:00:57.885412  1295 rpcz_store.cc:276] 1120 03:00:54.343773 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.343917 (+   144us) service_pool.cc:224] Handling call
1120 03:00:57.817764 (+3473847us) ts_tablet_manager.cc:1197] Deleting tablet ce612f9a121b4b18908724d2726e156f
1120 03:00:57.817803 (+    39us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:57.869146 (+ 51343us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:57.878025 (+  8879us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:57.885053 (+  7028us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3473236}
I20261120 03:00:57.886754   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet ce612f9a121b4b18908724d2726e156f (table default.Table_d2bd3060da1545548f422436c9c7c22e [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]) successfully deleted
I20261120 03:00:57.888249  1180 ts_tablet_manager.cc:1480] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.032s	user 0.006s	sys 0.000s
I20261120 03:00:57.890344  1500 raft_consensus.cc:534] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:57.890811  1500 raft_consensus.cc:556] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.892719  1500 leader_election.cc:290] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:57.894114  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a6ae8007455d4be3b50672b9c6b0fd8f" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:57.895620   857 leader_election.cc:343] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:57.896929  1181 tablet_bootstrap.cc:492] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:57.898110  1181 ts_tablet_manager.cc:1449] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.051s	user 0.011s	sys 0.001s
I20261120 03:00:57.898983  1180 tablet_bootstrap.cc:492] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:57.899432  1295 tablet_replica.cc:333] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:57.900235  1295 raft_consensus.cc:2440] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
W20261120 03:00:57.900468   857 leader_election.cc:343] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: a6ae8007455d4be3b50672b9c6b0fd8f
I20261120 03:00:57.900805   857 leader_election.cc:304] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:57.901448  1500 raft_consensus.cc:2966] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:57.900813  1295 raft_consensus.cc:2469] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:57.903885  1295 ts_tablet_manager.cc:1962] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:57.906102   496 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:45640 (request call id 154) took 2828 ms. Trace:
I20261120 03:00:57.906412   496 rpcz_store.cc:276] 1120 03:00:55.077675 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:56.822819 (+1745144us) service_pool.cc:224] Handling call
1120 03:00:57.040185 (+217366us) spinlock_profiling.cc:245] Waited 213 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07efa6 00007f155abf71f4 00007f155abf7182 00007f155abf70b1 00007f155abf702c 00007f155abf62e2 00007f1558e990bc 00007f1558e984f6
1120 03:00:57.062356 (+ 22171us) catalog_manager.cc:2145] Acquired catalog manager lock
1120 03:00:57.063850 (+  1494us) catalog_manager.cc:2191] Created new table and tablet info
1120 03:00:57.905483 (+841633us) catalog_manager.cc:2251] Wrote table and tablets to system table
1120 03:00:57.905763 (+   280us) catalog_manager.cc:2285] Inserted table and tablets into CatalogManager maps
1120 03:00:57.906016 (+   253us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:57.304357 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:57.304810 (+   453us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:57.304831 (+    21us) write_op.cc:440] Acquired schema lock
1120 03:00:57.304843 (+    12us) tablet.cc:710] Decoding operations
1120 03:00:57.305155 (+   312us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:57.305195 (+    40us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:57.305214 (+    19us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:00:57.305432 (+   218us) tablet.cc:758] Row locks acquired
1120 03:00:57.305446 (+    14us) write_op.cc:260] PREPARE: finished
1120 03:00:57.305568 (+   122us) write_op.cc:270] Start()
1120 03:00:57.305652 (+    84us) write_op.cc:276] Timestamp: P: 1795143657305507 usec, L: 0
1120 03:00:57.305669 (+    17us) op_driver.cc:350] REPLICATION: starting
1120 03:00:57.323025 (+ 17356us) log.cc:844] Serialized 433 byte log entry
1120 03:00:57.902209 (+579184us) op_driver.cc:466] REPLICATION: finished
1120 03:00:57.902604 (+   395us) write_op.cc:301] APPLY: starting
1120 03:00:57.902659 (+    55us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:57.902791 (+   132us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:57.902902 (+   111us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:57.903373 (+   471us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:57.903487 (+   114us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
1120 03:00:57.903526 (+    39us) write_op.cc:312] APPLY: finished
1120 03:00:57.903677 (+   151us) log.cc:844] Serialized 29 byte log entry
1120 03:00:57.903796 (+   119us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:57.903846 (+    50us) write_op.cc:459] Released schema lock
1120 03:00:57.903878 (+    32us) write_op.cc:346] FINISH: Updating metrics
1120 03:00:57.904076 (+   198us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:57.904099 (+    23us) write_op.cc:459] Released schema lock
Metrics: {"spinlock_wait_cycles":488951680,"child_traces":[["op",{"apply.queue_time_us":239,"apply.run_cpu_time_us":1440,"apply.run_wall_time_us":1440,"num_ops":2,"prepare.queue_time_us":239303,"prepare.run_cpu_time_us":2234,"prepare.run_wall_time_us":203991,"replication_time_us":596358,"spinlock_wait_cycles":7040}]]}
I20261120 03:00:57.909631  1181 raft_consensus.cc:366] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.910241  1181 raft_consensus.cc:392] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:57.910468  1181 raft_consensus.cc:787] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:57.911146  1181 consensus_queue.cc:261] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:57.915249  1181 ts_tablet_manager.cc:1480] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.016s	user 0.006s	sys 0.000s
I20261120 03:00:57.916774  1181 tablet_bootstrap.cc:492] T 5c5c0ac869a1419fad9f61ec647027ee P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:57.922274   500 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:45640 (request call id 158) took 2122 ms. Trace:
I20261120 03:00:57.922582   500 rpcz_store.cc:276] 1120 03:00:55.800226 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:57.433964 (+1633738us) service_pool.cc:224] Handling call
1120 03:00:57.454150 (+ 20186us) spinlock_profiling.cc:245] Waited 19.1 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07efa6 00007f155abf71f4 00007f155abf7182 00007f155abf70b1 00007f155abf702c 00007f155abf62e2 00007f1558e990bc 00007f1558e984f6
1120 03:00:57.456123 (+  1973us) catalog_manager.cc:2145] Acquired catalog manager lock
1120 03:00:57.457991 (+  1868us) catalog_manager.cc:2191] Created new table and tablet info
1120 03:00:57.921637 (+463646us) catalog_manager.cc:2251] Wrote table and tablets to system table
1120 03:00:57.921919 (+   282us) catalog_manager.cc:2285] Inserted table and tablets into CatalogManager maps
1120 03:00:57.922219 (+   300us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:57.781706 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:57.782173 (+   467us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:57.782198 (+    25us) write_op.cc:440] Acquired schema lock
1120 03:00:57.782212 (+    14us) tablet.cc:710] Decoding operations
1120 03:00:57.782504 (+   292us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:57.782543 (+    39us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:57.782567 (+    24us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:00:57.782782 (+   215us) tablet.cc:758] Row locks acquired
1120 03:00:57.782796 (+    14us) write_op.cc:260] PREPARE: finished
1120 03:00:57.782912 (+   116us) write_op.cc:270] Start()
1120 03:00:57.782986 (+    74us) write_op.cc:276] Timestamp: P: 1795143657782861 usec, L: 0
1120 03:00:57.782999 (+    13us) op_driver.cc:350] REPLICATION: starting
1120 03:00:57.811388 (+ 28389us) log.cc:844] Serialized 433 byte log entry
1120 03:00:57.919586 (+108198us) op_driver.cc:466] REPLICATION: finished
1120 03:00:57.919900 (+   314us) write_op.cc:301] APPLY: starting
1120 03:00:57.919951 (+    51us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:57.920054 (+   103us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:57.920168 (+   114us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:57.920590 (+   422us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:57.920719 (+   129us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
1120 03:00:57.920764 (+    45us) write_op.cc:312] APPLY: finished
1120 03:00:57.920933 (+   169us) log.cc:844] Serialized 29 byte log entry
1120 03:00:57.921055 (+   122us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:57.921129 (+    74us) write_op.cc:459] Released schema lock
1120 03:00:57.921160 (+    31us) write_op.cc:346] FINISH: Updating metrics
1120 03:00:57.921379 (+   219us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:57.921402 (+    23us) write_op.cc:459] Released schema lock
Metrics: {"spinlock_wait_cycles":43846016,"child_traces":[["op",{"apply.queue_time_us":186,"apply.run_cpu_time_us":1442,"apply.run_wall_time_us":1443,"num_ops":2,"prepare.queue_time_us":276316,"prepare.run_cpu_time_us":2083,"prepare.run_wall_time_us":42105,"replication_time_us":136479}]]}
I20261120 03:00:57.935593  1295 ts_tablet_manager.cc:1975] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:57.936021   899 tablet_service.cc:1495] Processing CreateTablet for tablet 59cc37f7ef9c409493bd89d0d2097339 (DEFAULT_TABLE table=default.Table_c1e302c0d12d4655b7871a1792728e02 [id=8ba1a0e7581846038d1e6bb366c52556]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:57.936517  1295 log.cc:1199] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/a08aa5ddcf1a4061a70275a8eecb1856
I20261120 03:00:57.936986   899 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 59cc37f7ef9c409493bd89d0d2097339. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:57.937348  1295 ts_tablet_manager.cc:1996] T a08aa5ddcf1a4061a70275a8eecb1856 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:57.939287  1295 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 88) took 3614 ms. Trace:
I20261120 03:00:57.939627  1295 rpcz_store.cc:276] 1120 03:00:54.324511 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.324732 (+   221us) service_pool.cc:224] Handling call
1120 03:00:57.899290 (+3574558us) ts_tablet_manager.cc:1197] Deleting tablet a08aa5ddcf1a4061a70275a8eecb1856
1120 03:00:57.899330 (+    40us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:57.916053 (+ 16723us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:57.935481 (+ 19428us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:57.939232 (+  3751us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3471693}
I20261120 03:00:57.940696  1295 tablet_replica.cc:333] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:57.941473  1295 raft_consensus.cc:2440] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:57.941943   901 tablet_service.cc:1495] Processing CreateTablet for tablet 1faba22e8e38418383b86e42e904a761 (DEFAULT_TABLE table=default.Table_47101d55d8f6485c816a02a54fda67a8 [id=252d229ff1334cd99ec99e5c921031bb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:57.942461  1295 raft_consensus.cc:2469] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:57.942878   901 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1faba22e8e38418383b86e42e904a761. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:57.945274  1181 tablet_bootstrap.cc:654] T 5c5c0ac869a1419fad9f61ec647027ee P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:57.953647  1051 tablet_service.cc:1495] Processing CreateTablet for tablet 59cc37f7ef9c409493bd89d0d2097339 (DEFAULT_TABLE table=default.Table_c1e302c0d12d4655b7871a1792728e02 [id=8ba1a0e7581846038d1e6bb366c52556]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:57.953768   753 tablet_service.cc:1495] Processing CreateTablet for tablet 59cc37f7ef9c409493bd89d0d2097339 (DEFAULT_TABLE table=default.Table_c1e302c0d12d4655b7871a1792728e02 [id=8ba1a0e7581846038d1e6bb366c52556]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:57.954752   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 59cc37f7ef9c409493bd89d0d2097339. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:57.954775  1051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 59cc37f7ef9c409493bd89d0d2097339. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:57.959353  1050 tablet_service.cc:1495] Processing CreateTablet for tablet 1faba22e8e38418383b86e42e904a761 (DEFAULT_TABLE table=default.Table_47101d55d8f6485c816a02a54fda67a8 [id=252d229ff1334cd99ec99e5c921031bb]), partition=RANGE (key) PARTITION UNBOUNDED
W20261120 03:00:57.960634   488 catalog_manager.cc:4863] Async tablet task AlterTable RPC for tablet 5da3d2724dd34aed88e6b08983188004 (table default.Table_d2bd3060da1545548f422436c9c7c22e [id=f5d860e17cd74d6aa7b08fbbb0f8c75a], current schema version=2) failed: Not found: failed to reset TS proxy: No leader found: Tablet 5da3d2724dd34aed88e6b08983188004 consensus state has no leader
I20261120 03:00:57.960675  1295 ts_tablet_manager.cc:1962] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:57.960948  1050 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1faba22e8e38418383b86e42e904a761. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:57.968905  1324 tablet_bootstrap.cc:492] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:57.971822   494 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet d736d65c3e0b419084b4934dad83f16a (table default.Table_04ddab7f706b43e7ba12fa247b3ca933 [id=9f571a69327a498e89f323c9ac0eee34]). Expected version 1 got 0
I20261120 03:00:57.972757   494 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 9882c9cbfbd640a49e3e3c8904ffc1e1 (table default.Table_0879bd445c234e4faefff97cd088035d [id=53fb1849755c4debad2dbfbf37271fbd]). Expected version 1 got 0
I20261120 03:00:57.982896  1324 tablet_bootstrap.cc:654] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:57.990805   486 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.124:38133: connect: Connection refused (error 111) [suppressed 5 similar messages]
I20261120 03:00:57.993474  1324 tablet_bootstrap.cc:492] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:57.993911  1324 ts_tablet_manager.cc:1449] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.025s	user 0.009s	sys 0.000s
I20261120 03:00:57.996981   753 tablet_service.cc:1495] Processing CreateTablet for tablet 1faba22e8e38418383b86e42e904a761 (DEFAULT_TABLE table=default.Table_47101d55d8f6485c816a02a54fda67a8 [id=252d229ff1334cd99ec99e5c921031bb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:57.997229  1324 raft_consensus.cc:366] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:57.997803  1324 raft_consensus.cc:392] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:57.998149  1048 tablet_service.cc:1495] Processing CreateTablet for tablet 3bc0f2f9cb7945ab9b0ead61c934f9dd (DEFAULT_TABLE table=default.Table_ab53c180c3b447daad84fe3018f841cc [id=66a7a958f6e94d4d802531bcb9f3c1a9]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:57.999289  1048 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3bc0f2f9cb7945ab9b0ead61c934f9dd. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.004662  1392 ts_tablet_manager.cc:1975] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:58.005081  1392 log.cc:1199] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/d753dd170cab4f18980e85707c4093e9
I20261120 03:00:58.005802  1392 ts_tablet_manager.cc:1996] T d753dd170cab4f18980e85707c4093e9 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:57.998044  1324 raft_consensus.cc:787] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:57.998994   752 tablet_service.cc:1495] Processing CreateTablet for tablet 3bc0f2f9cb7945ab9b0ead61c934f9dd (DEFAULT_TABLE table=default.Table_ab53c180c3b447daad84fe3018f841cc [id=66a7a958f6e94d4d802531bcb9f3c1a9]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:57.999763   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1faba22e8e38418383b86e42e904a761. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.008029  1324 consensus_queue.cc:261] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:58.008535   752 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3bc0f2f9cb7945ab9b0ead61c934f9dd. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.010878  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 80) took 4152 ms. Trace:
I20261120 03:00:58.011162  1392 rpcz_store.cc:276] 1120 03:00:53.858317 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:53.858612 (+   295us) service_pool.cc:224] Handling call
1120 03:00:57.742927 (+3884315us) ts_tablet_manager.cc:1197] Deleting tablet d753dd170cab4f18980e85707c4093e9
1120 03:00:57.742995 (+    68us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:57.875966 (+132971us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:58.004539 (+128573us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:58.010821 (+  6282us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3883761}
I20261120 03:00:58.012431   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet d753dd170cab4f18980e85707c4093e9 (table default.Table_269fbc3b47a44895be4b7bebddeee096 [id=62fab6f5923b4aa3be379f3d3d8d3046]) successfully deleted
I20261120 03:00:57.946784   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet a08aa5ddcf1a4061a70275a8eecb1856 (table default.Table_3c79e29085ad4875874397eaefcab250 [id=47e54c07e54d40adb2ac6907cb03bb68]) successfully deleted
W20261120 03:00:58.018293   487 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 71525.3us
I20261120 03:00:58.020164   900 tablet_service.cc:1495] Processing CreateTablet for tablet 3bc0f2f9cb7945ab9b0ead61c934f9dd (DEFAULT_TABLE table=default.Table_ab53c180c3b447daad84fe3018f841cc [id=66a7a958f6e94d4d802531bcb9f3c1a9]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.021272   900 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3bc0f2f9cb7945ab9b0ead61c934f9dd. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.022058  1181 tablet_bootstrap.cc:492] T 5c5c0ac869a1419fad9f61ec647027ee P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:58.022485  1181 ts_tablet_manager.cc:1449] T 5c5c0ac869a1419fad9f61ec647027ee P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.107s	user 0.013s	sys 0.001s
I20261120 03:00:58.024785  1181 raft_consensus.cc:366] T 5c5c0ac869a1419fad9f61ec647027ee P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.027288  1324 ts_tablet_manager.cc:1480] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.033s	user 0.002s	sys 0.004s
I20261120 03:00:58.025400  1181 raft_consensus.cc:392] T 5c5c0ac869a1419fad9f61ec647027ee P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:58.034523  1181 raft_consensus.cc:787] T 5c5c0ac869a1419fad9f61ec647027ee P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:58.036305  1181 consensus_queue.cc:261] T 5c5c0ac869a1419fad9f61ec647027ee P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.038614  1181 ts_tablet_manager.cc:1480] T 5c5c0ac869a1419fad9f61ec647027ee P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.016s	user 0.005s	sys 0.001s
I20261120 03:00:57.973721   494 catalog_manager.cc:5856] T fbbe19dadb08455285078e239d81bcca P 5beeefe8bef54caeac463d05d8c30fa6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65). New cstate: current_term: 1 leader_uuid: "5beeefe8bef54caeac463d05d8c30fa6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: UNKNOWN } } }
I20261120 03:00:58.040727   494 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
I20261120 03:00:58.042009   494 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 0db7db25d779459ebde532f938493975 (table default.Table_faa63585fa0a40f7b9fe1e97441cdc7a [id=f3b5f7a4e5494e82a472d0025df96e0c]). Expected version 1 got 0
I20261120 03:00:58.043166  1181 tablet_bootstrap.cc:492] T a08aa5ddcf1a4061a70275a8eecb1856 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:58.047468  1595 raft_consensus.cc:534] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:58.047909  1595 raft_consensus.cc:556] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.058746  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9dadc05b726346309d6b350f9abb4abb" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:58.060962   709 leader_election.cc:343] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:58.061285   898 tablet_service.cc:1495] Processing CreateTablet for tablet 0179a59749a14168a840e9d4e610be1f (DEFAULT_TABLE table=default.Table_cc56bf0ad8344f1a94d5da6e4d77e8c0 [id=b9034a083c7e4133a746b336335d3e42]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.062378   898 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0179a59749a14168a840e9d4e610be1f. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.067155  1045 tablet_service.cc:1495] Processing CreateTablet for tablet 0179a59749a14168a840e9d4e610be1f (DEFAULT_TABLE table=default.Table_cc56bf0ad8344f1a94d5da6e4d77e8c0 [id=b9034a083c7e4133a746b336335d3e42]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.068221  1045 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0179a59749a14168a840e9d4e610be1f. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.070509  1324 tablet_bootstrap.cc:492] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:58.075685   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9dadc05b726346309d6b350f9abb4abb" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:58.076085  1324 tablet_bootstrap.cc:654] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:58.077158   710 leader_election.cc:343] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:58.077530   710 leader_election.cc:304] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:58.078213  1597 raft_consensus.cc:2966] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:58.078434  1595 leader_election.cc:290] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:58.078783  1597 raft_consensus.cc:534] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:58.079186  1597 raft_consensus.cc:556] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.081004  1597 leader_election.cc:290] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:58.082497   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a04da9cb8f004443aa380902ea5427c5" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:00:58.084472   710 leader_election.cc:343] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:58.085104  1180 tablet_bootstrap.cc:654] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:58.085973  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a04da9cb8f004443aa380902ea5427c5" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:58.087466  1181 tablet_bootstrap.cc:654] T a08aa5ddcf1a4061a70275a8eecb1856 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:58.088339   709 leader_election.cc:343] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:58.088686   709 leader_election.cc:304] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:58.090044  1485 raft_consensus.cc:2966] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:58.091284   753 tablet_service.cc:1495] Processing CreateTablet for tablet 0179a59749a14168a840e9d4e610be1f (DEFAULT_TABLE table=default.Table_cc56bf0ad8344f1a94d5da6e4d77e8c0 [id=b9034a083c7e4133a746b336335d3e42]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.091864  1055 tablet_service.cc:1495] Processing CreateTablet for tablet ad49c8e00aca4a88a091baceae320477 (DEFAULT_TABLE table=default.Table_a121a467a36d4d159b7b3e108e3a00ea [id=a2382fc8065243b38b3ccff5ce8bec99]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.092293   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0179a59749a14168a840e9d4e610be1f. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.092967  1055 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ad49c8e00aca4a88a091baceae320477. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.100941  1324 tablet_bootstrap.cc:492] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:58.103874  1392 tablet_replica.cc:333] T 5c5c0ac869a1419fad9f61ec647027ee P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:58.104795  1392 raft_consensus.cc:2440] T 5c5c0ac869a1419fad9f61ec647027ee P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:58.105327  1392 raft_consensus.cc:2469] T 5c5c0ac869a1419fad9f61ec647027ee P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:58.043169   494 catalog_manager.cc:5856] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65). New cstate: current_term: 1 leader_uuid: "5beeefe8bef54caeac463d05d8c30fa6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: HEALTHY } } }
I20261120 03:00:58.110878  1324 ts_tablet_manager.cc:1449] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.041s	user 0.008s	sys 0.002s
I20261120 03:00:58.113107  1324 raft_consensus.cc:366] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:58.113698  1324 raft_consensus.cc:392] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:58.119652  1392 ts_tablet_manager.cc:1962] T 5c5c0ac869a1419fad9f61ec647027ee P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:58.120386  1181 tablet_bootstrap.cc:492] T a08aa5ddcf1a4061a70275a8eecb1856 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:58.126453  1181 ts_tablet_manager.cc:1449] T a08aa5ddcf1a4061a70275a8eecb1856 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.083s	user 0.008s	sys 0.005s
I20261120 03:00:58.130249   752 tablet_service.cc:1495] Processing CreateTablet for tablet ad49c8e00aca4a88a091baceae320477 (DEFAULT_TABLE table=default.Table_a121a467a36d4d159b7b3e108e3a00ea [id=a2382fc8065243b38b3ccff5ce8bec99]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.131352   752 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ad49c8e00aca4a88a091baceae320477. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.132812  1181 raft_consensus.cc:366] T a08aa5ddcf1a4061a70275a8eecb1856 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.133494  1181 raft_consensus.cc:392] T a08aa5ddcf1a4061a70275a8eecb1856 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:58.133776  1181 raft_consensus.cc:787] T a08aa5ddcf1a4061a70275a8eecb1856 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:58.134428  1181 consensus_queue.cc:261] T a08aa5ddcf1a4061a70275a8eecb1856 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.136111   501 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
I20261120 03:00:58.137074   501 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 9882c9cbfbd640a49e3e3c8904ffc1e1 (table default.Table_0879bd445c234e4faefff97cd088035d [id=53fb1849755c4debad2dbfbf37271fbd]). Expected version 1 got 0
I20261120 03:00:58.137858   501 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet d736d65c3e0b419084b4934dad83f16a (table default.Table_04ddab7f706b43e7ba12fa247b3ca933 [id=9f571a69327a498e89f323c9ac0eee34]). Expected version 1 got 0
I20261120 03:00:58.113917  1324 raft_consensus.cc:787] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:58.139195  1324 consensus_queue.cc:261] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:58.147181  1181 ts_tablet_manager.cc:1480] T a08aa5ddcf1a4061a70275a8eecb1856 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.020s	user 0.004s	sys 0.000s
I20261120 03:00:58.138239  1051 tablet_service.cc:1495] Processing CreateTablet for tablet ba85dab0d238487181ce5c1592a2c7fd (DEFAULT_TABLE table=default.Table_a97b43669b2d4944ad7c61d64e5a2cbd [id=f83c29744a484e5f9a76851039bb1a4d]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.156558  1324 ts_tablet_manager.cc:1480] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.045s	user 0.005s	sys 0.000s
I20261120 03:00:58.161768  1181 tablet_bootstrap.cc:492] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:58.163794  1295 ts_tablet_manager.cc:1975] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:58.164175  1295 log.cc:1199] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/88b9e9692a874ae993dde4b9e2586a70
I20261120 03:00:58.165828  1295 ts_tablet_manager.cc:1996] T 88b9e9692a874ae993dde4b9e2586a70 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:58.167320  1324 tablet_bootstrap.cc:492] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:58.171939  1051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ba85dab0d238487181ce5c1592a2c7fd. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.146502   497 catalog_manager.cc:5856] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 reported cstate change: term changed from 0 to 1, leader changed from <none> to 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67). New cstate: current_term: 1 leader_uuid: "823abdbd90ba47bbacad0f8dcff78234" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: HEALTHY } } }
I20261120 03:00:58.176745  1181 tablet_bootstrap.cc:654] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:58.182264  1295 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 90) took 3688 ms. Trace:
I20261120 03:00:58.182579  1295 rpcz_store.cc:276] 1120 03:00:54.496976 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.497284 (+   308us) service_pool.cc:224] Handling call
1120 03:00:57.940569 (+3443285us) ts_tablet_manager.cc:1197] Deleting tablet 88b9e9692a874ae993dde4b9e2586a70
1120 03:00:57.940605 (+    36us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:57.991451 (+ 50846us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:58.163676 (+172225us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:58.182201 (+ 18525us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3442692}
I20261120 03:00:58.191319   497 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 9882c9cbfbd640a49e3e3c8904ffc1e1 (table default.Table_0879bd445c234e4faefff97cd088035d [id=53fb1849755c4debad2dbfbf37271fbd]). Expected version 1 got 0
I20261120 03:00:58.193604  1295 tablet_replica.cc:333] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:58.197321  1392 ts_tablet_manager.cc:1975] T 5c5c0ac869a1419fad9f61ec647027ee P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:58.197813  1392 log.cc:1199] T 5c5c0ac869a1419fad9f61ec647027ee P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/5c5c0ac869a1419fad9f61ec647027ee
I20261120 03:00:58.198511  1392 ts_tablet_manager.cc:1996] T 5c5c0ac869a1419fad9f61ec647027ee P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:58.201534  1295 raft_consensus.cc:2440] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:58.202047  1324 tablet_bootstrap.cc:654] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:58.221369  1324 tablet_bootstrap.cc:492] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:58.221903  1324 ts_tablet_manager.cc:1449] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.055s	user 0.006s	sys 0.009s
I20261120 03:00:58.225378  1070 raft_consensus.cc:3296] T 168cb98bb1b242a5a0377bf292204d96 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:58.227108  1324 raft_consensus.cc:366] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:58.227751  1324 raft_consensus.cc:392] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:58.228015  1324 raft_consensus.cc:787] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:58.228784  1324 consensus_queue.cc:261] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:58.233767  1324 ts_tablet_manager.cc:1480] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.012s	user 0.004s	sys 0.000s
I20261120 03:00:58.237542  1324 tablet_bootstrap.cc:492] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:58.240675   752 tablet_service.cc:1495] Processing CreateTablet for tablet ba85dab0d238487181ce5c1592a2c7fd (DEFAULT_TABLE table=default.Table_a97b43669b2d4944ad7c61d64e5a2cbd [id=f83c29744a484e5f9a76851039bb1a4d]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.241813   752 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ba85dab0d238487181ce5c1592a2c7fd. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.242376  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 81) took 4151 ms. Trace:
I20261120 03:00:58.242697  1392 rpcz_store.cc:276] 1120 03:00:54.091037 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.093163 (+  2126us) service_pool.cc:224] Handling call
1120 03:00:58.103710 (+4010547us) ts_tablet_manager.cc:1197] Deleting tablet 5c5c0ac869a1419fad9f61ec647027ee
1120 03:00:58.103767 (+    57us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:58.158187 (+ 54420us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:58.197169 (+ 38982us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:58.242306 (+ 45137us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":4007368}
I20261120 03:00:58.243873   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 88b9e9692a874ae993dde4b9e2586a70 (table default.Table_e4c5ff8f28ae4efca67257942a4fcb77 [id=b30488f6d14b4c9ab1566f2376be602a]) successfully deleted
I20261120 03:00:58.245934  1324 tablet_bootstrap.cc:654] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:58.246748   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 5c5c0ac869a1419fad9f61ec647027ee (table default.Table_5c5911c8746d4b26843ec77c8141d3b6 [id=ffb2040a2e114893855d4d8c66dfa855]) successfully deleted
I20261120 03:00:58.247532   902 tablet_service.cc:1495] Processing CreateTablet for tablet ad49c8e00aca4a88a091baceae320477 (DEFAULT_TABLE table=default.Table_a121a467a36d4d159b7b3e108e3a00ea [id=a2382fc8065243b38b3ccff5ce8bec99]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.248622   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ad49c8e00aca4a88a091baceae320477. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.252918  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 82) took 3991 ms. Trace:
I20261120 03:00:58.253243  1392 rpcz_store.cc:276] 1120 03:00:54.261550 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.261674 (+   124us) service_pool.cc:224] Handling call
1120 03:00:58.243584 (+3981910us) ts_tablet_manager.cc:1197] Deleting tablet 5f5c8711a6eb43c3aff1c27ed5115d6a
1120 03:00:58.246378 (+  2794us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:58.252861 (+  6483us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3979661}
W20261120 03:00:58.254721   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 5f5c8711a6eb43c3aff1c27ed5115d6a because the tablet was not found. No further retry: Not found: Tablet not found: 5f5c8711a6eb43c3aff1c27ed5115d6a
I20261120 03:00:58.255777  1392 tablet_replica.cc:333] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:58.255637   903 tablet_service.cc:1495] Processing CreateTablet for tablet ba85dab0d238487181ce5c1592a2c7fd (DEFAULT_TABLE table=default.Table_a97b43669b2d4944ad7c61d64e5a2cbd [id=f83c29744a484e5f9a76851039bb1a4d]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.256611   903 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ba85dab0d238487181ce5c1592a2c7fd. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.256695  1392 raft_consensus.cc:2440] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:58.257164  1392 raft_consensus.cc:2469] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:58.259037  1392 ts_tablet_manager.cc:1962] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:58.262238  1295 raft_consensus.cc:2469] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:58.268975  1295 ts_tablet_manager.cc:1962] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:58.279362  1392 ts_tablet_manager.cc:1975] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:58.282238  1392 log.cc:1199] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/136056e5242a42ef8fa4b40c989b5423
I20261120 03:00:58.283389  1392 ts_tablet_manager.cc:1996] T 136056e5242a42ef8fa4b40c989b5423 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
W20261120 03:00:58.283977  1012 consensus_peers.cc:597] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 -> Peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Couldn't send request to peer 3d30c2428730481db9b0ffd279dd3440. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ce612f9a121b4b18908724d2726e156f. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:58.286037  1181 tablet_bootstrap.cc:492] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:58.286479  1181 ts_tablet_manager.cc:1449] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.125s	user 0.014s	sys 0.004s
I20261120 03:00:58.289515  1181 raft_consensus.cc:366] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.293556  1181 raft_consensus.cc:392] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:58.294341  1181 raft_consensus.cc:787] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:58.296793  1181 consensus_queue.cc:261] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.303684  1181 ts_tablet_manager.cc:1480] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.017s	user 0.003s	sys 0.003s
I20261120 03:00:58.306363  1181 tablet_bootstrap.cc:492] T c6fea104598e4f6d876f96675f2da816 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:58.315774  1048 tablet_service.cc:1495] Processing CreateTablet for tablet 73c993944b0a4977b55d40ad63bb69e3 (DEFAULT_TABLE table=default.Table_43f3d907270e4093a84918b0a505ecc9 [id=1cdd4bbccdbf4449ba67b9aa7089a6fd]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.317034  1048 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 73c993944b0a4977b55d40ad63bb69e3. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.318207  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 83) took 4053 ms. Trace:
I20261120 03:00:58.318605  1392 rpcz_store.cc:276] 1120 03:00:54.264719 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.264839 (+   120us) service_pool.cc:224] Handling call
1120 03:00:58.255528 (+3990689us) ts_tablet_manager.cc:1197] Deleting tablet 136056e5242a42ef8fa4b40c989b5423
1120 03:00:58.255654 (+   126us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:58.267719 (+ 12065us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:58.279252 (+ 11533us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:58.318151 (+ 38899us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3990103}
I20261120 03:00:58.319692  1392 tablet_replica.cc:333] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:58.320350   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 136056e5242a42ef8fa4b40c989b5423 (table default.Table_d2ca6a35f8b842d784167fd22339644a [id=afefca6af8d64812b4f3e37317a1c8d7]) successfully deleted
I20261120 03:00:58.320421  1392 raft_consensus.cc:2440] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:58.320914  1392 raft_consensus.cc:2469] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:58.322855  1392 ts_tablet_manager.cc:1962] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:58.336651   495 catalog_manager.cc:5856] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65). New cstate: current_term: 1 leader_uuid: "5beeefe8bef54caeac463d05d8c30fa6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: UNKNOWN } } }
I20261120 03:00:58.313963  1181 tablet_bootstrap.cc:654] T c6fea104598e4f6d876f96675f2da816 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:58.343369   753 tablet_service.cc:1495] Processing CreateTablet for tablet 73c993944b0a4977b55d40ad63bb69e3 (DEFAULT_TABLE table=default.Table_43f3d907270e4093a84918b0a505ecc9 [id=1cdd4bbccdbf4449ba67b9aa7089a6fd]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.344404   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 73c993944b0a4977b55d40ad63bb69e3. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.346832   756 tablet_service.cc:1495] Processing CreateTablet for tablet 9e42a416be884341994d2d5395286798 (DEFAULT_TABLE table=default.Table_e4c5ff8f28ae4efca67257942a4fcb77 [id=b30488f6d14b4c9ab1566f2376be602a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.347848   756 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9e42a416be884341994d2d5395286798. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.356470  1324 tablet_bootstrap.cc:492] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:58.356870  1324 ts_tablet_manager.cc:1449] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.119s	user 0.019s	sys 0.019s
I20261120 03:00:58.359021  1324 raft_consensus.cc:366] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:58.359606  1324 raft_consensus.cc:392] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:58.359844  1324 raft_consensus.cc:787] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:58.360530  1324 consensus_queue.cc:261] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:58.361363  1055 tablet_service.cc:1495] Processing CreateTablet for tablet 00f7c7d95bae4d55be6b437701989ba3 (DEFAULT_TABLE table=default.Table_fbcff6b6a1aa454aaf2cd73f3b7381b0 [id=1ee32758557140be8ff0bdcb71ab4f8e]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.362478  1055 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00f7c7d95bae4d55be6b437701989ba3. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.368543  1324 ts_tablet_manager.cc:1480] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.011s	user 0.006s	sys 0.001s
I20261120 03:00:58.372491  1324 tablet_bootstrap.cc:492] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:58.386797  1737 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.96.124:38133 every 8 connection(s)
I20261120 03:00:58.337890   495 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet d736d65c3e0b419084b4934dad83f16a (table default.Table_04ddab7f706b43e7ba12fa247b3ca933 [id=9f571a69327a498e89f323c9ac0eee34]). Expected version 1 got 0
I20261120 03:00:58.387780   495 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 9882c9cbfbd640a49e3e3c8904ffc1e1 (table default.Table_0879bd445c234e4faefff97cd088035d [id=53fb1849755c4debad2dbfbf37271fbd]). Expected version 1 got 0
I20261120 03:00:58.391054  1300 rpc_server.cc:307] RPC server started. Bound to: 127.0.96.124:38133
I20261120 03:00:58.392210  1051 tablet_service.cc:1495] Processing CreateTablet for tablet 9e42a416be884341994d2d5395286798 (DEFAULT_TABLE table=default.Table_e4c5ff8f28ae4efca67257942a4fcb77 [id=b30488f6d14b4c9ab1566f2376be602a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.398904  1300 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/master-2/data/info.pb
I20261120 03:00:58.408380   495 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
I20261120 03:00:58.409539  1051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9e42a416be884341994d2d5395286798. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.410797   495 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 0db7db25d779459ebde532f938493975 (table default.Table_faa63585fa0a40f7b9fe1e97441cdc7a [id=f3b5f7a4e5494e82a472d0025df96e0c]). Expected version 1 got 0
I20261120 03:00:58.415645   899 tablet_service.cc:1495] Processing CreateTablet for tablet 00f7c7d95bae4d55be6b437701989ba3 (DEFAULT_TABLE table=default.Table_fbcff6b6a1aa454aaf2cd73f3b7381b0 [id=1ee32758557140be8ff0bdcb71ab4f8e]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.416738   899 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00f7c7d95bae4d55be6b437701989ba3. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.430410  1500 raft_consensus.cc:534] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:58.434674  1500 raft_consensus.cc:556] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.436411  1180 tablet_bootstrap.cc:492] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:58.436946  1180 ts_tablet_manager.cc:1449] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.538s	user 0.012s	sys 0.005s
I20261120 03:00:58.436900  1500 leader_election.cc:290] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:58.434173   898 tablet_service.cc:1495] Processing CreateTablet for tablet 73c993944b0a4977b55d40ad63bb69e3 (DEFAULT_TABLE table=default.Table_43f3d907270e4093a84918b0a505ecc9 [id=1cdd4bbccdbf4449ba67b9aa7089a6fd]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.441843   898 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 73c993944b0a4977b55d40ad63bb69e3. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:00:58.447872   710 consensus_peers.cc:597] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Couldn't send request to peer 3d30c2428730481db9b0ffd279dd3440. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 5c5c0ac869a1419fad9f61ec647027ee. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:58.449220  1070 raft_consensus.cc:3296] T a08aa5ddcf1a4061a70275a8eecb1856 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:58.450629  1324 tablet_bootstrap.cc:654] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:58.442329   900 tablet_service.cc:1495] Processing CreateTablet for tablet 9e42a416be884341994d2d5395286798 (DEFAULT_TABLE table=default.Table_e4c5ff8f28ae4efca67257942a4fcb77 [id=b30488f6d14b4c9ab1566f2376be602a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.460525   900 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9e42a416be884341994d2d5395286798. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.439352  1180 raft_consensus.cc:366] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.466359  1180 raft_consensus.cc:392] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:58.466718  1180 raft_consensus.cc:787] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:58.467432  1180 consensus_queue.cc:261] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.472283  1180 ts_tablet_manager.cc:1480] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.035s	user 0.005s	sys 0.000s
I20261120 03:00:58.479758  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3d89fb1d7fa14ae8ac252c68338ed4d6" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:58.481869   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3d89fb1d7fa14ae8ac252c68338ed4d6" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:58.482584   776 raft_consensus.cc:2062] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:58.495054  1739 sys_catalog.cc:263] Verifying existing consensus state
I20261120 03:00:58.513916  1739 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f: Bootstrap starting.
I20261120 03:00:58.489646   857 leader_election.cc:304] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
W20261120 03:00:58.515931   857 leader_election.cc:343] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:58.517403  1735 raft_consensus.cc:3024] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:58.517758  1735 raft_consensus.cc:534] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:58.518062  1735 raft_consensus.cc:3296] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:58.534961  1180 tablet_bootstrap.cc:492] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:58.556087  1597 raft_consensus.cc:534] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:58.556550  1597 raft_consensus.cc:556] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:58.558501  1597 leader_election.cc:290] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:58.558993  1125 heartbeater.cc:345] Connected to a master server at 127.0.96.124:38133
I20261120 03:00:58.579753  1324 tablet_bootstrap.cc:492] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:58.580205  1324 ts_tablet_manager.cc:1449] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.208s	user 0.009s	sys 0.000s
I20261120 03:00:58.587425  1324 raft_consensus.cc:366] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:58.588107  1324 raft_consensus.cc:392] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:58.588392  1324 raft_consensus.cc:787] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:58.592150  1567 raft_consensus.cc:534] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:58.597333   756 tablet_service.cc:1495] Processing CreateTablet for tablet 00f7c7d95bae4d55be6b437701989ba3 (DEFAULT_TABLE table=default.Table_fbcff6b6a1aa454aaf2cd73f3b7381b0 [id=1ee32758557140be8ff0bdcb71ab4f8e]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.598346   756 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00f7c7d95bae4d55be6b437701989ba3. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.606261  1567 raft_consensus.cc:556] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.606329  1324 consensus_queue.cc:261] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:58.608865  1567 leader_election.cc:290] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
W20261120 03:00:58.609838  1011 leader_election.cc:343] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: a6ae8007455d4be3b50672b9c6b0fd8f
I20261120 03:00:58.610399   823 heartbeater.cc:345] Connected to a master server at 127.0.96.124:38133
I20261120 03:00:58.624915  1324 ts_tablet_manager.cc:1480] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.044s	user 0.004s	sys 0.005s
I20261120 03:00:58.626503   753 tablet_service.cc:1495] Processing CreateTablet for tablet 5da3d2724dd34aed88e6b08983188004 (DEFAULT_TABLE table=default.Table_d2bd3060da1545548f422436c9c7c22e [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.627933   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5da3d2724dd34aed88e6b08983188004. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.628069  1324 tablet_bootstrap.cc:492] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:58.647667  1597 raft_consensus.cc:534] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:58.648121  1597 raft_consensus.cc:556] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.650015  1597 leader_election.cc:290] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
W20261120 03:00:58.651959   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 129547us
I20261120 03:00:58.652840   901 tablet_service.cc:1495] Processing CreateTablet for tablet 5da3d2724dd34aed88e6b08983188004 (DEFAULT_TABLE table=default.Table_d2bd3060da1545548f422436c9c7c22e [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.653863   901 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5da3d2724dd34aed88e6b08983188004. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.655228  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a1192a7f050a4f5387a30b2216dec5f0" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:58.658432  1739 log.cc:826] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f: Log is configured to *not* fsync() on all Append() calls
W20261120 03:00:58.660672   709 leader_election.cc:343] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:58.663028   498 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet d736d65c3e0b419084b4934dad83f16a (table default.Table_04ddab7f706b43e7ba12fa247b3ca933 [id=9f571a69327a498e89f323c9ac0eee34]). Expected version 1 got 0
I20261120 03:00:58.663668  1048 tablet_service.cc:1495] Processing CreateTablet for tablet 5da3d2724dd34aed88e6b08983188004 (DEFAULT_TABLE table=default.Table_d2bd3060da1545548f422436c9c7c22e [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.663928   498 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 9882c9cbfbd640a49e3e3c8904ffc1e1 (table default.Table_0879bd445c234e4faefff97cd088035d [id=53fb1849755c4debad2dbfbf37271fbd]). Expected version 1 got 0
I20261120 03:00:58.665100   498 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
I20261120 03:00:58.666971   498 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 0db7db25d779459ebde532f938493975 (table default.Table_faa63585fa0a40f7b9fe1e97441cdc7a [id=f3b5f7a4e5494e82a472d0025df96e0c]). Expected version 1 got 0
I20261120 03:00:58.679258  1048 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5da3d2724dd34aed88e6b08983188004. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.672673  1181 tablet_bootstrap.cc:492] T c6fea104598e4f6d876f96675f2da816 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:58.680719  1181 ts_tablet_manager.cc:1449] T c6fea104598e4f6d876f96675f2da816 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.376s	user 0.010s	sys 0.002s
I20261120 03:00:58.683337  1181 raft_consensus.cc:366] T c6fea104598e4f6d876f96675f2da816 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.684273  1181 raft_consensus.cc:392] T c6fea104598e4f6d876f96675f2da816 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:58.684587  1181 raft_consensus.cc:787] T c6fea104598e4f6d876f96675f2da816 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:58.685272  1181 consensus_queue.cc:261] T c6fea104598e4f6d876f96675f2da816 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.685827   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet d736d65c3e0b419084b4934dad83f16a (table default.Table_04ddab7f706b43e7ba12fa247b3ca933 [id=9f571a69327a498e89f323c9ac0eee34]). Expected version 1 got 0
I20261120 03:00:58.687420   500 catalog_manager.cc:5856] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66). New cstate: current_term: 1 leader_uuid: "3d30c2428730481db9b0ffd279dd3440" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } }
I20261120 03:00:58.688395   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 0db7db25d779459ebde532f938493975 (table default.Table_faa63585fa0a40f7b9fe1e97441cdc7a [id=f3b5f7a4e5494e82a472d0025df96e0c]). Expected version 1 got 0
I20261120 03:00:58.689006   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 9882c9cbfbd640a49e3e3c8904ffc1e1 (table default.Table_0879bd445c234e4faefff97cd088035d [id=53fb1849755c4debad2dbfbf37271fbd]). Expected version 1 got 0
I20261120 03:00:58.689870  1595 raft_consensus.cc:534] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:58.690295  1595 raft_consensus.cc:556] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:58.692061  1595 leader_election.cc:290] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:58.694059  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1faba22e8e38418383b86e42e904a761" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:58.696223  1567 raft_consensus.cc:534] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:58.696641  1567 raft_consensus.cc:556] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.698499  1567 leader_election.cc:290] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:58.651762  1735 raft_consensus.cc:556] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.700615   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "88b9e9692a874ae993dde4b9e2586a70" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:58.702397  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3d89fb1d7fa14ae8ac252c68338ed4d6" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:58.703873   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3d89fb1d7fa14ae8ac252c68338ed4d6" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
W20261120 03:00:58.706631   709 leader_election.cc:343] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:58.707646   917 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1faba22e8e38418383b86e42e904a761" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:00:58.710815   857 leader_election.cc:343] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:58.704408   776 raft_consensus.cc:3296] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:58.712291   776 raft_consensus.cc:2062] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
W20261120 03:00:58.714156   710 leader_election.cc:343] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:58.715996   710 leader_election.cc:304] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:58.717145  1485 raft_consensus.cc:2966] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:58.674890  1070 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b549a06d15db449b851fe9549a963416" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:58.725174   709 leader_election.cc:343] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:58.725636   857 leader_election.cc:304] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:58.727689  1595 raft_consensus.cc:534] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:58.727638   903 tablet_service.cc:1495] Processing CreateTablet for tablet 9ea5f0b680bd401bbc2b69d780e1ef50 (DEFAULT_TABLE table=default.Table_25c78056a3a44c77bb4bd8bf7a471fe5 [id=8033f4439236415497110763392dddb0]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.728114  1595 raft_consensus.cc:556] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:58.719826   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b549a06d15db449b851fe9549a963416" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:58.730160  1748 raft_consensus.cc:3024] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:58.730609  1748 raft_consensus.cc:744] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:00:58.731352  1748 consensus_queue.cc:238] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.731986  1595 leader_election.cc:290] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:58.700843  1735 leader_election.cc:290] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
W20261120 03:00:58.736331   710 leader_election.cc:343] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:58.736735   710 leader_election.cc:304] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:58.744725  1597 raft_consensus.cc:2966] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:58.407354   385 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu as pid 1300
I20261120 03:00:58.750463  1180 tablet_bootstrap.cc:654] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:58.759316  1045 tablet_service.cc:1495] Processing CreateTablet for tablet 9ea5f0b680bd401bbc2b69d780e1ef50 (DEFAULT_TABLE table=default.Table_25c78056a3a44c77bb4bd8bf7a471fe5 [id=8033f4439236415497110763392dddb0]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.760637  1045 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9ea5f0b680bd401bbc2b69d780e1ef50. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:00:58.764135  1012 leader_election.cc:343] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Not found: Tablet not found: 88b9e9692a874ae993dde4b9e2586a70
I20261120 03:00:58.764568  1012 leader_election.cc:304] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:58.765574  1567 raft_consensus.cc:2966] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:58.765913   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a1192a7f050a4f5387a30b2216dec5f0" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:58.766090   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "59cc37f7ef9c409493bd89d0d2097339" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:58.767252   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a6ae8007455d4be3b50672b9c6b0fd8f" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:58.767930   919 raft_consensus.cc:2062] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:00:58.768968  1012 leader_election.cc:304] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 5beeefe8bef54caeac463d05d8c30fa6
W20261120 03:00:58.769888   710 leader_election.cc:343] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:58.770063  1567 raft_consensus.cc:3024] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:58.770278   710 leader_election.cc:304] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:58.770417  1567 raft_consensus.cc:534] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:58.770752  1567 raft_consensus.cc:3296] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:58.771037  1485 raft_consensus.cc:2966] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:58.771771  1567 raft_consensus.cc:556] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:58.772151   710 leader_election.cc:343] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:58.786460  1011 leader_election.cc:343] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: a6ae8007455d4be3b50672b9c6b0fd8f
I20261120 03:00:58.787439  1567 leader_election.cc:290] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:58.789119   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a6ae8007455d4be3b50672b9c6b0fd8f" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:00:58.789992  1485 raft_consensus.cc:534] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:58.791124  1485 raft_consensus.cc:556] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:58.793808  1070 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "59cc37f7ef9c409493bd89d0d2097339" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:58.795534   709 leader_election.cc:343] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:58.795936   709 leader_election.cc:304] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:58.796892  1485 leader_election.cc:290] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:58.798596  1485 raft_consensus.cc:2966] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:58.807149   753 tablet_service.cc:1495] Processing CreateTablet for tablet 9ea5f0b680bd401bbc2b69d780e1ef50 (DEFAULT_TABLE table=default.Table_25c78056a3a44c77bb4bd8bf7a471fe5 [id=8033f4439236415497110763392dddb0]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.808269   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9ea5f0b680bd401bbc2b69d780e1ef50. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.812638  1392 ts_tablet_manager.cc:1975] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:58.813015  1392 log.cc:1199] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/9a31431495274c648625b75900b4ac47
I20261120 03:00:58.813665  1392 ts_tablet_manager.cc:1996] T 9a31431495274c648625b75900b4ac47 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:58.817431   970 heartbeater.cc:345] Connected to a master server at 127.0.96.124:38133
I20261120 03:00:58.818816   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ad49c8e00aca4a88a091baceae320477" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:58.789662   919 raft_consensus.cc:3296] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:58.822391   919 raft_consensus.cc:2062] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
I20261120 03:00:58.825749  1181 ts_tablet_manager.cc:1480] T c6fea104598e4f6d876f96675f2da816 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.145s	user 0.006s	sys 0.000s
I20261120 03:00:58.825762   752 tablet_service.cc:1495] Processing CreateTablet for tablet 29e6a50c9e1f4b778737abdef9ad073a (DEFAULT_TABLE table=default.Table_c1e41dc463834422869aec7e3bace0f1 [id=0d5e41d23488417f94be9626cc967dbc]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.826745  1181 tablet_bootstrap.cc:492] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:58.826895   752 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 29e6a50c9e1f4b778737abdef9ad073a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.839095  1012 leader_election.cc:304] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:58.840570  1381 raft_consensus.cc:3024] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:58.841136  1381 raft_consensus.cc:744] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Becoming Leader. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Running, Role: LEADER
I20261120 03:00:58.841928  1381 consensus_queue.cc:238] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.846253  1295 ts_tablet_manager.cc:1975] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:58.846678  1295 log.cc:1199] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/68179488b9bc4e03b65a9f1d434c95b5
I20261120 03:00:58.847362  1295 ts_tablet_manager.cc:1996] T 68179488b9bc4e03b65a9f1d434c95b5 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:58.848122   902 tablet_service.cc:1495] Processing CreateTablet for tablet 29e6a50c9e1f4b778737abdef9ad073a (DEFAULT_TABLE table=default.Table_c1e41dc463834422869aec7e3bace0f1 [id=0d5e41d23488417f94be9626cc967dbc]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.850559  1070 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ad49c8e00aca4a88a091baceae320477" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:58.852103   709 leader_election.cc:343] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:58.855340   496 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
I20261120 03:00:58.856030   496 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 0db7db25d779459ebde532f938493975 (table default.Table_faa63585fa0a40f7b9fe1e97441cdc7a [id=f3b5f7a4e5494e82a472d0025df96e0c]). Expected version 1 got 0
I20261120 03:00:58.858361   496 catalog_manager.cc:5856] T 168cb98bb1b242a5a0377bf292204d96 P 5beeefe8bef54caeac463d05d8c30fa6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65). New cstate: current_term: 1 leader_uuid: "5beeefe8bef54caeac463d05d8c30fa6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: HEALTHY } } }
I20261120 03:00:58.859952   496 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet d736d65c3e0b419084b4934dad83f16a (table default.Table_04ddab7f706b43e7ba12fa247b3ca933 [id=9f571a69327a498e89f323c9ac0eee34]). Expected version 1 got 0
W20261120 03:00:58.860999   710 leader_election.cc:343] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:58.861394   710 leader_election.cc:304] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:58.862252  1595 raft_consensus.cc:2966] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:58.862869   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 29e6a50c9e1f4b778737abdef9ad073a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.864550  1055 tablet_service.cc:1495] Processing CreateTablet for tablet 29e6a50c9e1f4b778737abdef9ad073a (DEFAULT_TABLE table=default.Table_c1e41dc463834422869aec7e3bace0f1 [id=0d5e41d23488417f94be9626cc967dbc]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.866533   903 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9ea5f0b680bd401bbc2b69d780e1ef50. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.868157  1324 tablet_bootstrap.cc:654] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:58.871575  1055 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 29e6a50c9e1f4b778737abdef9ad073a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.879777   899 tablet_service.cc:1495] Processing CreateTablet for tablet 4da5eb2f2f114cc792488c5c74bb1414 (DEFAULT_TABLE table=default.Table_db0de8e58235444da0d82057b21e8b0f [id=684e9a19b696496296849afef03970c2]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.880928   899 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4da5eb2f2f114cc792488c5c74bb1414. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.888659  1595 raft_consensus.cc:534] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:58.889160  1595 raft_consensus.cc:556] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:58.877496  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 84) took 4610 ms. Trace:
I20261120 03:00:58.882009  1567 consensus_queue.cc:1049] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:58.891360  1181 tablet_bootstrap.cc:654] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:58.892343  1392 rpcz_store.cc:276] 1120 03:00:54.267056 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.267186 (+   130us) service_pool.cc:224] Handling call
1120 03:00:58.319530 (+4052344us) ts_tablet_manager.cc:1197] Deleting tablet 9a31431495274c648625b75900b4ac47
1120 03:00:58.319598 (+    68us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:58.585135 (+265537us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:58.812506 (+227371us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:58.877434 (+ 64928us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":4051839}
I20261120 03:00:58.893877  1392 tablet_replica.cc:333] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:58.894573  1392 raft_consensus.cc:2440] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:58.895017  1392 raft_consensus.cc:2469] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:58.896788   756 tablet_service.cc:1495] Processing CreateTablet for tablet 4da5eb2f2f114cc792488c5c74bb1414 (DEFAULT_TABLE table=default.Table_db0de8e58235444da0d82057b21e8b0f [id=684e9a19b696496296849afef03970c2]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.897799   756 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4da5eb2f2f114cc792488c5c74bb1414. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.898671  1180 tablet_bootstrap.cc:492] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:58.899115  1180 ts_tablet_manager.cc:1449] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.364s	user 0.009s	sys 0.003s
I20261120 03:00:58.902541   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3bc0f2f9cb7945ab9b0ead61c934f9dd" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:58.902907  1051 tablet_service.cc:1495] Processing CreateTablet for tablet 4da5eb2f2f114cc792488c5c74bb1414 (DEFAULT_TABLE table=default.Table_db0de8e58235444da0d82057b21e8b0f [id=684e9a19b696496296849afef03970c2]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.904029  1051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4da5eb2f2f114cc792488c5c74bb1414. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.908257  1595 leader_election.cc:290] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:58.910038  1070 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3bc0f2f9cb7945ab9b0ead61c934f9dd" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:58.911381   709 leader_election.cc:343] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:00:58.912529   710 leader_election.cc:343] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:58.912957   710 leader_election.cc:304] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:58.896023   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 9a31431495274c648625b75900b4ac47 (table default.Table_3bb077af5e15449b85500aca3a1500c3 [id=234b77d1cf1847b0ae8afa8d3dbdff71]) successfully deleted
I20261120 03:00:58.884233   496 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 9882c9cbfbd640a49e3e3c8904ffc1e1 (table default.Table_0879bd445c234e4faefff97cd088035d [id=53fb1849755c4debad2dbfbf37271fbd]). Expected version 1 got 0
I20261120 03:00:58.917603  1295 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 102) took 2077 ms. Trace:
I20261120 03:00:58.917984  1295 rpcz_store.cc:276] 1120 03:00:56.840563 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:56.851851 (+ 11288us) service_pool.cc:224] Handling call
1120 03:00:58.193442 (+1341591us) ts_tablet_manager.cc:1197] Deleting tablet 68179488b9bc4e03b65a9f1d434c95b5
1120 03:00:58.193482 (+    40us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:58.391931 (+198449us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:58.846126 (+454195us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:58.917548 (+ 71422us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":125585,"tablet-delete.queue_time_us":1307034}
W20261120 03:00:58.918650   710 consensus_peers.cc:597] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Couldn't send request to peer 3d30c2428730481db9b0ffd279dd3440. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a08aa5ddcf1a4061a70275a8eecb1856. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:58.919615  1295 tablet_replica.cc:333] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:58.919387   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 68179488b9bc4e03b65a9f1d434c95b5 (table default.Table_650dc67e0ac74c149a3f64d456767938 [id=cfcd5fea97fb4ab3baff99eb28cd9476]) successfully deleted
I20261120 03:00:58.921379   496 catalog_manager.cc:5856] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65). New cstate: current_term: 1 leader_uuid: "5beeefe8bef54caeac463d05d8c30fa6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: HEALTHY } } }
I20261120 03:00:58.924333  1595 raft_consensus.cc:2966] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:58.926970  1747 raft_consensus.cc:534] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:58.927390  1747 raft_consensus.cc:556] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.929397  1747 leader_election.cc:290] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:58.930037  1180 raft_consensus.cc:366] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.930658  1180 raft_consensus.cc:392] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:58.930923  1180 raft_consensus.cc:787] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:58.931885  1045 tablet_service.cc:1495] Processing CreateTablet for tablet 26f8ba69516c447dbd91e0d87cd519d3 (DEFAULT_TABLE table=default.Table_90a09b39479a4615b0dccfc155f2b7e3 [id=b49ab89508bc4849bb929955e316aec6]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.933037  1045 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 26f8ba69516c447dbd91e0d87cd519d3. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:00:58.934204   857 consensus_peers.cc:597] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:58.934530   898 tablet_service.cc:1495] Processing CreateTablet for tablet 26f8ba69516c447dbd91e0d87cd519d3 (DEFAULT_TABLE table=default.Table_90a09b39479a4615b0dccfc155f2b7e3 [id=b49ab89508bc4849bb929955e316aec6]), partition=RANGE (key) PARTITION UNBOUNDED
W20261120 03:00:58.935817  1011 consensus_peers.cc:597] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 -> Peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Couldn't send request to peer 5beeefe8bef54caeac463d05d8c30fa6. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a6ae8007455d4be3b50672b9c6b0fd8f. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:58.936363   898 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 26f8ba69516c447dbd91e0d87cd519d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.942468  1070 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "014bd9c83be34c31b90fe4bcf31f1649" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:58.944133  1324 tablet_bootstrap.cc:492] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:58.944567  1324 ts_tablet_manager.cc:1449] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.317s	user 0.006s	sys 0.005s
I20261120 03:00:58.946838  1324 raft_consensus.cc:366] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:58.947425  1324 raft_consensus.cc:392] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
W20261120 03:00:58.950620  1012 consensus_peers.cc:597] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 -> Peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Couldn't send request to peer 3d30c2428730481db9b0ffd279dd3440. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: STOPPING. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:58.955014   753 tablet_service.cc:1495] Processing CreateTablet for tablet 26f8ba69516c447dbd91e0d87cd519d3 (DEFAULT_TABLE table=default.Table_90a09b39479a4615b0dccfc155f2b7e3 [id=b49ab89508bc4849bb929955e316aec6]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:58.956116   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 26f8ba69516c447dbd91e0d87cd519d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:58.960269  1392 ts_tablet_manager.cc:1962] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
W20261120 03:00:58.962231   857 leader_election.cc:343] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:58.966142  1295 raft_consensus.cc:2440] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
W20261120 03:00:58.967288   857 leader_election.cc:343] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: 014bd9c83be34c31b90fe4bcf31f1649
I20261120 03:00:58.967695   857 leader_election.cc:304] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:58.968022   499 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet c6a65c1a4a574961bec657ad7c9048e0
I20261120 03:00:58.969588   499 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:45640 (request call id 159) took 1934 ms. Trace:
I20261120 03:00:58.969413  1180 consensus_queue.cc:261] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:58.970990  1050 tablet_service.cc:1542] Processing DeleteTablet for tablet c6a65c1a4a574961bec657ad7c9048e0 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:57 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:58.971588   752 tablet_service.cc:1542] Processing DeleteTablet for tablet c6a65c1a4a574961bec657ad7c9048e0 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:57 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:58.969882   499 rpcz_store.cc:276] 1120 03:00:57.035564 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:57.437927 (+402363us) service_pool.cc:224] Handling call
1120 03:00:57.450699 (+ 12772us) spinlock_profiling.cc:245] Waited 7.95 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07f2b6 00007f155abfbcf4 00007f155abfbc82 00007f155abfbbb1 00007f155abfbb2c 00007f155abfade2 00007f1558e990bc 00007f1558e984f6
1120 03:00:57.451809 (+  1110us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:00:57.453078 (+  1269us) catalog_manager.cc:2859] Modifying in-memory table state
1120 03:00:57.453207 (+   129us) catalog_manager.cc:2868] Locking tablets
1120 03:00:57.930764 (+477557us) catalog_manager.cc:2882] Removing table and tablets from system table
1120 03:00:57.977904 (+ 47140us) catalog_manager.cc:2900] Removing table from by-name map
1120 03:00:57.978104 (+   200us) catalog_manager.cc:2916] Committing in-memory state
1120 03:00:58.967903 (+989799us) catalog_manager.cc:2920] Aborting table tasks
1120 03:00:58.969531 (+  1628us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:57.944117 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:57.944497 (+   380us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:57.944522 (+    25us) write_op.cc:440] Acquired schema lock
1120 03:00:57.944535 (+    13us) tablet.cc:710] Decoding operations
1120 03:00:57.944848 (+   313us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:57.944889 (+    41us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:57.944910 (+    21us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:00:57.945185 (+   275us) tablet.cc:758] Row locks acquired
1120 03:00:57.945198 (+    13us) write_op.cc:260] PREPARE: finished
1120 03:00:57.945307 (+   109us) write_op.cc:270] Start()
1120 03:00:57.945383 (+    76us) write_op.cc:276] Timestamp: P: 1795143657945245 usec, L: 0
1120 03:00:57.945398 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:00:57.945868 (+   470us) log.cc:844] Serialized 725 byte log entry
1120 03:00:57.975883 (+ 30015us) op_driver.cc:466] REPLICATION: finished
1120 03:00:57.976196 (+   313us) write_op.cc:301] APPLY: starting
1120 03:00:57.976248 (+    52us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:57.976352 (+   104us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:57.976439 (+    87us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:57.976684 (+   245us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:57.976813 (+   129us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
1120 03:00:57.976856 (+    43us) write_op.cc:312] APPLY: finished
1120 03:00:57.977033 (+   177us) log.cc:844] Serialized 29 byte log entry
1120 03:00:57.977636 (+   603us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:57.977704 (+    68us) write_op.cc:459] Released schema lock
1120 03:00:57.977734 (+    30us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"spinlock_wait_cycles":18277120,"child_traces":[["op",{"apply.queue_time_us":173,"apply.run_cpu_time_us":1293,"apply.run_wall_time_us":2052,"num_ops":2,"prepare.queue_time_us":11602,"prepare.run_cpu_time_us":1987,"prepare.run_wall_time_us":1987,"replication_time_us":30380}]]}
W20261120 03:00:58.974627   499 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.66:36141 (request call id 27) took 1078 ms (client timeout 1000 ms). Trace:
W20261120 03:00:58.975070   499 rpcz_store.cc:269] 1120 03:00:57.896187 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:58.974301 (+1078114us) service_pool.cc:211] Skipping call since client already timed out
1120 03:00:58.974568 (+   267us) inbound_call.cc:177] Queueing failure response
Metrics: {}
I20261120 03:00:58.985872   501 catalog_manager.cc:6409] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: default.Table_a1f866996679431e9e1c4c5f6f8cdd12 [id=a275e215155b4824a1494f0954577fe8] alter complete (version 1)
W20261120 03:00:58.999847   497 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.67:53803 (request call id 17) took 2495 ms (client timeout 1000 ms). Trace:
W20261120 03:00:59.000286   497 rpcz_store.cc:269] 1120 03:00:56.504683 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:56.790959 (+286276us) service_pool.cc:224] Handling call
1120 03:00:57.028780 (+237821us) spinlock_profiling.cc:245] Waited 234 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07d9ce 00007f155abf0174 00007f155abf0102 00007f155abf0031 00007f155abeffac 00007f155abef262 00007f1558e990bc 00007f1558e984f6
1120 03:00:58.241392 (+1212612us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:58.241416 (+    24us) write_op.cc:459] Released schema lock
1120 03:00:58.999783 (+758367us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:58.193335 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:58.193748 (+   413us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:58.193771 (+    23us) write_op.cc:440] Acquired schema lock
1120 03:00:58.193784 (+    13us) tablet.cc:710] Decoding operations
1120 03:00:58.194105 (+   321us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:58.194144 (+    39us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:58.194165 (+    21us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:00:58.194323 (+   158us) tablet.cc:758] Row locks acquired
1120 03:00:58.194335 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:00:58.194446 (+   111us) write_op.cc:270] Start()
1120 03:00:58.194526 (+    80us) write_op.cc:276] Timestamp: P: 1795143658194399 usec, L: 0
1120 03:00:58.194540 (+    14us) op_driver.cc:350] REPLICATION: starting
1120 03:00:58.195024 (+   484us) log.cc:844] Serialized 527 byte log entry
1120 03:00:58.205945 (+ 10921us) op_driver.cc:466] REPLICATION: finished
1120 03:00:58.206395 (+   450us) write_op.cc:301] APPLY: starting
1120 03:00:58.206449 (+    54us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:58.206541 (+    92us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:58.206631 (+    90us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:58.206801 (+   170us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:58.206920 (+   119us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:58.206962 (+    42us) write_op.cc:312] APPLY: finished
1120 03:00:58.207139 (+   177us) log.cc:844] Serialized 23 byte log entry
1120 03:00:58.207274 (+   135us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:58.207332 (+    58us) write_op.cc:459] Released schema lock
1120 03:00:58.207361 (+    29us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"reported_tablets":16,"spinlock_wait_cycles":538470784,"child_traces":[["op",{"apply.queue_time_us":313,"apply.run_cpu_time_us":1126,"apply.run_wall_time_us":1125,"num_ops":1,"prepare.queue_time_us":91,"prepare.run_cpu_time_us":2046,"prepare.run_wall_time_us":2047,"raft.queue_time_us":511,"raft.run_cpu_time_us":743,"raft.run_wall_time_us":742,"replication_time_us":11309}]]}
I20261120 03:00:59.018030   502 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet b549a06d15db449b851fe9549a963416
I20261120 03:00:59.019586   502 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:45640 (request call id 142) took 4487 ms. Trace:
I20261120 03:00:59.019685   754 tablet_service.cc:1542] Processing DeleteTablet for tablet b549a06d15db449b851fe9549a963416 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:54 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:59.019865   502 rpcz_store.cc:276] 1120 03:00:54.532362 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.536364 (+  4002us) service_pool.cc:224] Handling call
1120 03:00:54.537671 (+  1307us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:00:54.538219 (+   548us) catalog_manager.cc:2859] Modifying in-memory table state
1120 03:00:54.538355 (+   136us) catalog_manager.cc:2868] Locking tablets
1120 03:00:55.337134 (+798779us) catalog_manager.cc:2882] Removing table and tablets from system table
1120 03:00:56.806720 (+1469586us) catalog_manager.cc:2900] Removing table from by-name map
1120 03:00:56.806933 (+   213us) catalog_manager.cc:2916] Committing in-memory state
1120 03:00:59.017823 (+2210890us) catalog_manager.cc:2920] Aborting table tasks
1120 03:00:59.019520 (+  1697us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:56.182746 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:56.183172 (+   426us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:56.183193 (+    21us) write_op.cc:440] Acquired schema lock
1120 03:00:56.183205 (+    12us) tablet.cc:710] Decoding operations
1120 03:00:56.183553 (+   348us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:56.183586 (+    33us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:56.183606 (+    20us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:00:56.183901 (+   295us) tablet.cc:758] Row locks acquired
1120 03:00:56.183914 (+    13us) write_op.cc:260] PREPARE: finished
1120 03:00:56.184257 (+   343us) write_op.cc:270] Start()
1120 03:00:56.184338 (+    81us) write_op.cc:276] Timestamp: P: 1795143656184102 usec, L: 0
1120 03:00:56.184354 (+    16us) op_driver.cc:350] REPLICATION: starting
1120 03:00:56.184886 (+   532us) log.cc:844] Serialized 756 byte log entry
1120 03:00:56.803766 (+618880us) op_driver.cc:466] REPLICATION: finished
1120 03:00:56.804243 (+   477us) write_op.cc:301] APPLY: starting
1120 03:00:56.804349 (+   106us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:56.804457 (+   108us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:56.804557 (+   100us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:56.804880 (+   323us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:56.805015 (+   135us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
1120 03:00:56.805059 (+    44us) write_op.cc:312] APPLY: finished
1120 03:00:56.805242 (+   183us) log.cc:844] Serialized 29 byte log entry
1120 03:00:56.805848 (+   606us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:56.805909 (+    61us) write_op.cc:459] Released schema lock
1120 03:00:56.805940 (+    31us) write_op.cc:346] FINISH: Updating metrics
1120 03:00:56.807124 (+  1184us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:56.807149 (+    25us) write_op.cc:459] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":311,"apply.run_cpu_time_us":1449,"apply.run_wall_time_us":2849,"num_ops":2,"prepare.queue_time_us":832761,"prepare.run_cpu_time_us":2230,"prepare.run_wall_time_us":19993,"replication_time_us":619315,"spinlock_wait_cycles":4352}]]}
W20261120 03:00:59.031602   501 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.66:36141 (request call id 25) took 3925 ms (client timeout 1000 ms). Trace:
W20261120 03:00:59.032034   501 rpcz_store.cc:269] 1120 03:00:55.106136 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:55.310487 (+204351us) service_pool.cc:224] Handling call
1120 03:00:56.117670 (+807183us) spinlock_profiling.cc:245] Waited 804 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07d9ce 00007f155abf0174 00007f155abf0102 00007f155abf0031 00007f155abeffac 00007f155abef262 00007f1558e990bc 00007f1558e984f6
1120 03:00:59.031532 (+2913862us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:58.144118 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:58.144503 (+   385us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:58.144526 (+    23us) write_op.cc:440] Acquired schema lock
1120 03:00:58.144538 (+    12us) tablet.cc:710] Decoding operations
1120 03:00:58.144793 (+   255us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:58.144834 (+    41us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:58.144855 (+    21us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:00:58.144997 (+   142us) tablet.cc:758] Row locks acquired
1120 03:00:58.145010 (+    13us) write_op.cc:260] PREPARE: finished
1120 03:00:58.145131 (+   121us) write_op.cc:270] Start()
1120 03:00:58.145209 (+    78us) write_op.cc:276] Timestamp: P: 1795143658145037 usec, L: 0
1120 03:00:58.145225 (+    16us) op_driver.cc:350] REPLICATION: starting
1120 03:00:58.145675 (+   450us) log.cc:844] Serialized 356 byte log entry
1120 03:00:58.162557 (+ 16882us) op_driver.cc:466] REPLICATION: finished
1120 03:00:58.162944 (+   387us) write_op.cc:301] APPLY: starting
1120 03:00:58.162996 (+    52us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:58.163082 (+    86us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:58.163166 (+    84us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:58.163328 (+   162us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:58.163447 (+   119us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:58.163484 (+    37us) write_op.cc:312] APPLY: finished
1120 03:00:58.163671 (+   187us) log.cc:844] Serialized 23 byte log entry
1120 03:00:58.163802 (+   131us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:58.163862 (+    60us) write_op.cc:459] Released schema lock
1120 03:00:58.163891 (+    29us) write_op.cc:346] FINISH: Updating metrics
1120 03:00:58.164124 (+   233us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:58.164147 (+    23us) write_op.cc:459] Released schema lock
Metrics: {"reported_tablets":13,"spinlock_wait_cycles":1850105216,"child_traces":[["op",{"apply.queue_time_us":246,"apply.run_cpu_time_us":1118,"apply.run_wall_time_us":1117,"num_ops":1,"prepare.queue_time_us":89,"prepare.run_cpu_time_us":1880,"prepare.run_wall_time_us":1881,"raft.queue_time_us":3005,"raft.run_cpu_time_us":678,"raft.run_wall_time_us":678,"replication_time_us":17239}]]}
I20261120 03:00:59.035763   501 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_29554130703446b79df44400fb36c94e"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:59.037400   501 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_29554130703446b79df44400fb36c94e in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:59.064119  1048 tablet_service.cc:1542] Processing DeleteTablet for tablet d753dd170cab4f18980e85707c4093e9 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:59.068864  1048 tablet_service.cc:1542] Processing DeleteTablet for tablet 9a31431495274c648625b75900b4ac47 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:49 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:59.071283  1761 tablet_replica.cc:333] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:59.072113  1059 tablet_service.cc:1542] Processing DeleteTablet for tablet 020be7f7cda0467aab56f657cbe8d0ad with delete_type TABLET_DATA_DELETED (replaced by tablet 600a18b7e5104a78935d906b8adfde6f) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:59.072831  1761 raft_consensus.cc:2440] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:59.073043  1048 tablet_service.cc:1542] Processing DeleteTablet for tablet 200b234da897419a82bf5844620adf24 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:52 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:59.073282  1761 raft_consensus.cc:2469] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:59.065798  1050 tablet_service.cc:1542] Processing DeleteTablet for tablet a642c32c1b6043978516d97efa2347a3 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:59.073588  1046 tablet_service.cc:1542] Processing DeleteTablet for tablet c3b56d0ccab140c28a37808d9d5902c6 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:48 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:58.947657  1324 raft_consensus.cc:787] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:59.095495   904 tablet_service.cc:1542] Processing DeleteTablet for tablet b549a06d15db449b851fe9549a963416 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:54 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:59.096328   904 tablet_service.cc:1542] Processing DeleteTablet for tablet ce612f9a121b4b18908724d2726e156f with delete_type TABLET_DATA_DELETED (replaced by tablet 5da3d2724dd34aed88e6b08983188004) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:59.086323  1324 consensus_queue.cc:261] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:59.097594  1747 consensus_queue.cc:1049] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:59.104992   897 tablet_service.cc:1542] Processing DeleteTablet for tablet 020be7f7cda0467aab56f657cbe8d0ad with delete_type TABLET_DATA_DELETED (replaced by tablet 600a18b7e5104a78935d906b8adfde6f) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:59.105674   896 tablet_service.cc:1542] Processing DeleteTablet for tablet 68179488b9bc4e03b65a9f1d434c95b5 with delete_type TABLET_DATA_DELETED (replaced by tablet 4d79fc4c8f1c4a5f83c0dc94337c857e) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:59.106397   895 tablet_service.cc:1542] Processing DeleteTablet for tablet 1885397e42774a89bb347bc776284bc0 with delete_type TABLET_DATA_DELETED (replaced by tablet b549a06d15db449b851fe9549a963416) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:59.119393   901 tablet_service.cc:1542] Processing DeleteTablet for tablet c6a65c1a4a574961bec657ad7c9048e0 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:57 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:59.122517  1059 tablet_service.cc:1542] Processing DeleteTablet for tablet b549a06d15db449b851fe9549a963416 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:54 UTC) from {username='slave'} at 127.0.0.1:50744
W20261120 03:00:59.126626   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 159348us
I20261120 03:00:59.129144  1180 ts_tablet_manager.cc:1480] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.230s	user 0.005s	sys 0.000s
I20261120 03:00:59.052662  1739 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f: Bootstrap replayed 1/1 log segments. Stats: ops{read=9 overwritten=0 applied=9 ignored=0} inserts{seen=11 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20261120 03:00:59.130088  1180 tablet_bootstrap.cc:492] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:59.135682  1324 ts_tablet_manager.cc:1480] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.191s	user 0.005s	sys 0.000s
I20261120 03:00:59.136523  1324 tablet_bootstrap.cc:492] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:59.145552  1740 raft_consensus.cc:2966] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:59.146421  1740 raft_consensus.cc:534] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:59.146783  1740 raft_consensus.cc:556] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:59.148564  1740 leader_election.cc:290] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:59.149078  1761 ts_tablet_manager.cc:1962] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:59.150614   774 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "dc4133a8f33b4902a6308f9bdd4f91d2" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:59.151316   774 raft_consensus.cc:2062] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:00:59.153301   857 leader_election.cc:304] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:59.154692  1748 raft_consensus.cc:3024] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:00:59.155006  1748 raft_consensus.cc:534] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:59.155297  1748 raft_consensus.cc:3296] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:59.156054  1748 raft_consensus.cc:556] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:59.164932   501 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:45640 (request call id 160) took 2125 ms. Trace:
I20261120 03:00:59.165300   501 rpcz_store.cc:276] 1120 03:00:57.039144 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.035405 (+1996261us) service_pool.cc:224] Handling call
1120 03:00:59.075630 (+ 40225us) catalog_manager.cc:2145] Acquired catalog manager lock
1120 03:00:59.077032 (+  1402us) catalog_manager.cc:2191] Created new table and tablet info
1120 03:00:59.126352 (+ 49320us) catalog_manager.cc:2251] Wrote table and tablets to system table
1120 03:00:59.164587 (+ 38235us) catalog_manager.cc:2285] Inserted table and tablets into CatalogManager maps
1120 03:00:59.164870 (+   283us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:59.090698 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:59.091077 (+   379us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:59.091108 (+    31us) write_op.cc:440] Acquired schema lock
1120 03:00:59.091122 (+    14us) tablet.cc:710] Decoding operations
1120 03:00:59.091385 (+   263us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:59.091419 (+    34us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:59.091440 (+    21us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:00:59.091628 (+   188us) tablet.cc:758] Row locks acquired
1120 03:00:59.091641 (+    13us) write_op.cc:260] PREPARE: finished
1120 03:00:59.091751 (+   110us) write_op.cc:270] Start()
1120 03:00:59.091826 (+    75us) write_op.cc:276] Timestamp: P: 1795143659091677 usec, L: 0
1120 03:00:59.091840 (+    14us) op_driver.cc:350] REPLICATION: starting
1120 03:00:59.092295 (+   455us) log.cc:844] Serialized 433 byte log entry
1120 03:00:59.110670 (+ 18375us) op_driver.cc:466] REPLICATION: finished
1120 03:00:59.110992 (+   322us) write_op.cc:301] APPLY: starting
1120 03:00:59.111040 (+    48us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:59.111160 (+   120us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:59.111247 (+    87us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:59.111619 (+   372us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:59.111741 (+   122us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
1120 03:00:59.111790 (+    49us) write_op.cc:312] APPLY: finished
1120 03:00:59.111949 (+   159us) log.cc:844] Serialized 29 byte log entry
1120 03:00:59.112087 (+   138us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:59.112147 (+    60us) write_op.cc:459] Released schema lock
1120 03:00:59.112181 (+    34us) write_op.cc:346] FINISH: Updating metrics
1120 03:00:59.112376 (+   195us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:59.112400 (+    24us) write_op.cc:459] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":182,"apply.run_cpu_time_us":1355,"apply.run_wall_time_us":1355,"mutex_wait_us":11781,"num_ops":2,"prepare.queue_time_us":12556,"prepare.run_cpu_time_us":1784,"prepare.run_wall_time_us":1785,"replication_time_us":18740}]]}
I20261120 03:00:59.172089   501 master_service.cc:965] ReplaceTablet: received request to replace tablet 4abb67f3819a45b9b05dde04842e2c9d from {username='slave'} at 127.0.0.1:45640
I20261120 03:00:59.186560  1739 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f: Bootstrap complete.
I20261120 03:00:59.188297  1070 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "dc4133a8f33b4902a6308f9bdd4f91d2" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:00:59.190551   857 leader_election.cc:343] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:59.190864  1181 tablet_bootstrap.cc:492] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:59.191212  1180 tablet_bootstrap.cc:654] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:59.191262  1181 ts_tablet_manager.cc:1449] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.365s	user 0.018s	sys 0.016s
I20261120 03:00:59.201012  1068 raft_consensus.cc:3296] T c6fea104598e4f6d876f96675f2da816 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:59.202279  1324 tablet_bootstrap.cc:654] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:59.206602  1748 leader_election.cc:290] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:59.207834  1295 pending_rounds.cc:70] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440: Trying to abort 1 pending ops.
I20261120 03:00:59.208062  1295 pending_rounds.cc:77] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7352908426627227648 op_type: NO_OP noop_request { }
I20261120 03:00:59.208361  1295 raft_consensus.cc:3115] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: NO_OP replication failed: Aborted: Op aborted
I20261120 03:00:59.208689  1295 raft_consensus.cc:2469] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:59.210011  1070 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "dc4133a8f33b4902a6308f9bdd4f91d2" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
W20261120 03:00:59.211287   857 leader_election.cc:343] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:59.226625  1295 ts_tablet_manager.cc:1962] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:59.226644   774 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "dc4133a8f33b4902a6308f9bdd4f91d2" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:00:59.227193   774 raft_consensus.cc:3296] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:00:59.228011   774 raft_consensus.cc:2062] T dc4133a8f33b4902a6308f9bdd4f91d2 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:00:59.234259  1181 raft_consensus.cc:366] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:59.235008  1181 raft_consensus.cc:392] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:59.235318  1181 raft_consensus.cc:787] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:59.236050  1181 consensus_queue.cc:261] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:59.238107  1181 ts_tablet_manager.cc:1480] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.047s	user 0.005s	sys 0.000s
I20261120 03:00:59.239403  1485 raft_consensus.cc:534] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:59.239830  1485 raft_consensus.cc:556] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:59.241628  1485 leader_election.cc:290] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:59.243085   501 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 4abb67f3819a45b9b05dde04842e2c9d
I20261120 03:00:59.248481  1180 tablet_bootstrap.cc:492] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:59.248870  1180 ts_tablet_manager.cc:1449] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.119s	user 0.009s	sys 0.000s
I20261120 03:00:59.251343   756 tablet_service.cc:1542] Processing DeleteTablet for tablet 4abb67f3819a45b9b05dde04842e2c9d with delete_type TABLET_DATA_DELETED (replaced by tablet df90c4151aa44b83a12b17680dc161da) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:59.251389  1180 raft_consensus.cc:366] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:59.251960  1180 raft_consensus.cc:392] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:59.252213  1180 raft_consensus.cc:787] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:59.254653   501 catalog_manager.cc:6952] ReplaceTablet: tablet 4abb67f3819a45b9b05dde04842e2c9d deleted and replaced by tablet df90c4151aa44b83a12b17680dc161da
I20261120 03:00:59.260027   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0179a59749a14168a840e9d4e610be1f" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:00:59.263263   710 leader_election.cc:343] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:59.266249  1485 raft_consensus.cc:534] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 3d30c2428730481db9b0ffd279dd3440)
I20261120 03:00:59.266757  1485 raft_consensus.cc:556] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:59.272192  1180 consensus_queue.cc:261] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:59.277143   898 tablet_service.cc:1542] Processing DeleteTablet for tablet 4abb67f3819a45b9b05dde04842e2c9d with delete_type TABLET_DATA_DELETED (replaced by tablet df90c4151aa44b83a12b17680dc161da) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:59.279744   501 rpcz_store.cc:275] Call kudu.master.MasterService.ReplaceTablet from 127.0.0.1:45640 (request call id 161) took 2172 ms. Trace:
I20261120 03:00:59.250032  1045 tablet_service.cc:1542] Processing DeleteTablet for tablet 4abb67f3819a45b9b05dde04842e2c9d with delete_type TABLET_DATA_DELETED (replaced by tablet df90c4151aa44b83a12b17680dc161da) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:59.250653  1070 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0179a59749a14168a840e9d4e610be1f" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:59.269296  1078 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1885397e42774a89bb347bc776284bc0" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 2 candidate_status { last_received { term: 1 index: 1 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:59.285056  1078 raft_consensus.cc:2062] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
W20261120 03:00:59.286269   709 leader_election.cc:343] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:59.286597   709 leader_election.cc:304] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:59.288986  1181 tablet_bootstrap.cc:492] T dc4133a8f33b4902a6308f9bdd4f91d2 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:59.289008  1324 tablet_bootstrap.cc:492] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:59.289513  1324 ts_tablet_manager.cc:1449] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.153s	user 0.011s	sys 0.000s
I20261120 03:00:59.291821  1324 raft_consensus.cc:366] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:59.292397  1324 raft_consensus.cc:392] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:59.292654  1324 raft_consensus.cc:787] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:59.298159  1180 ts_tablet_manager.cc:1480] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.034s	user 0.008s	sys 0.000s
I20261120 03:00:59.298811  1485 leader_election.cc:290] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:59.299755  1485 raft_consensus.cc:2966] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:59.303004  1295 ts_tablet_manager.cc:1975] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:59.303378  1295 log.cc:1199] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/a6ae8007455d4be3b50672b9c6b0fd8f
I20261120 03:00:59.304011  1295 ts_tablet_manager.cc:1996] T a6ae8007455d4be3b50672b9c6b0fd8f P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
W20261120 03:00:59.304498   710 leader_election.cc:343] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Not found: Tablet not found: 1885397e42774a89bb347bc776284bc0
I20261120 03:00:59.309123   857 leader_election.cc:304] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:59.309572   502 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
I20261120 03:00:59.310149  1747 raft_consensus.cc:3024] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:00:59.310585  1747 raft_consensus.cc:744] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:00:59.310599   502 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet d736d65c3e0b419084b4934dad83f16a (table default.Table_04ddab7f706b43e7ba12fa247b3ca933 [id=9f571a69327a498e89f323c9ac0eee34]). Expected version 1 got 0
I20261120 03:00:59.311219   502 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 0db7db25d779459ebde532f938493975 (table default.Table_faa63585fa0a40f7b9fe1e97441cdc7a [id=f3b5f7a4e5494e82a472d0025df96e0c]). Expected version 1 got 0
I20261120 03:00:59.311352  1747 consensus_queue.cc:238] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:59.311851   502 catalog_manager.cc:5856] T 35f0ff39b36148e8abfdea3435a3b8b4 P 3d30c2428730481db9b0ffd279dd3440 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66). New cstate: current_term: 1 leader_uuid: "3d30c2428730481db9b0ffd279dd3440" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: HEALTHY } } }
I20261120 03:00:59.317431  1761 ts_tablet_manager.cc:1975] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:59.317817  1761 log.cc:1199] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/c6a65c1a4a574961bec657ad7c9048e0
I20261120 03:00:59.319675  1180 tablet_bootstrap.cc:492] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:59.281390  1125 heartbeater.cc:345] Connected to a master server at 127.0.96.124:38133
I20261120 03:00:59.297837  1381 raft_consensus.cc:534] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:59.322369  1381 raft_consensus.cc:556] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:59.324337  1381 leader_election.cc:290] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:59.282853   501 rpcz_store.cc:276] 1120 03:00:57.106854 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.171876 (+2065022us) service_pool.cc:224] Handling call
1120 03:00:59.238843 (+ 66967us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:59.239259 (+   416us) write_op.cc:459] Released schema lock
1120 03:00:59.279153 (+ 39894us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:59.205002 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:59.205387 (+   385us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:59.205412 (+    25us) write_op.cc:440] Acquired schema lock
1120 03:00:59.205424 (+    12us) tablet.cc:710] Decoding operations
1120 03:00:59.205721 (+   297us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:59.205764 (+    43us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:59.205785 (+    21us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:00:59.205979 (+   194us) tablet.cc:758] Row locks acquired
1120 03:00:59.205991 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:00:59.206118 (+   127us) write_op.cc:270] Start()
1120 03:00:59.206207 (+    89us) write_op.cc:276] Timestamp: P: 1795143659206057 usec, L: 0
1120 03:00:59.206222 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:00:59.206672 (+   450us) log.cc:844] Serialized 656 byte log entry
1120 03:00:59.215281 (+  8609us) op_driver.cc:466] REPLICATION: finished
1120 03:00:59.215728 (+   447us) write_op.cc:301] APPLY: starting
1120 03:00:59.215785 (+    57us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:59.215899 (+   114us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:59.215995 (+    96us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:59.216325 (+   330us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:59.216516 (+   191us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:59.216567 (+    51us) write_op.cc:312] APPLY: finished
1120 03:00:59.216743 (+   176us) log.cc:844] Serialized 29 byte log entry
1120 03:00:59.216880 (+   137us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:59.216947 (+    67us) write_op.cc:459] Released schema lock
1120 03:00:59.216977 (+    30us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"child_traces":[["op",{"apply.queue_time_us":274,"apply.run_cpu_time_us":1414,"apply.run_wall_time_us":1413,"num_ops":2,"prepare.queue_time_us":139,"prepare.run_cpu_time_us":1926,"prepare.run_wall_time_us":1926,"raft.queue_time_us":164,"raft.run_cpu_time_us":675,"raft.run_wall_time_us":675,"replication_time_us":8969}]]}
I20261120 03:00:59.313763   502 catalog_manager.cc:5856] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 3d30c2428730481db9b0ffd279dd3440 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66). New cstate: current_term: 1 leader_uuid: "3d30c2428730481db9b0ffd279dd3440" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: HEALTHY } } }
I20261120 03:00:59.331966  1324 consensus_queue.cc:261] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:59.337765   501 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_931d2c41a0834cf19b10a6d61e85698a" } new_table_name: "default.Table_92f6cc71382f4deba2472c49ae8f6a36" modify_external_catalogs: true
W20261120 03:00:59.340324  1012 leader_election.cc:343] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Not found: Tablet not found: 200b234da897419a82bf5844620adf24
I20261120 03:00:59.341497  1324 ts_tablet_manager.cc:1480] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.052s	user 0.002s	sys 0.004s
I20261120 03:00:59.344762  1051 tablet_service.cc:1495] Processing CreateTablet for tablet 0dca5a2820ad4197b83a804b0310e57c (DEFAULT_TABLE table=default.Table_331ec71dbe224231bb5e1cce1470d2ea [id=f3b2bd5e97a6498ca79bf605a3ce28f6]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.345909  1051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0dca5a2820ad4197b83a804b0310e57c. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:59.360100  1392 ts_tablet_manager.cc:1975] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:59.360455  1392 log.cc:1199] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/df61317717d3401eb74cd93aa336c4dc
I20261120 03:00:59.361032  1392 ts_tablet_manager.cc:1996] T df61317717d3401eb74cd93aa336c4dc P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
W20261120 03:00:59.368369   709 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 82124us
I20261120 03:00:59.369123   709 leader_election.cc:304] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440
I20261120 03:00:59.371526  1485 raft_consensus.cc:3024] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader pre-election won for term 2
I20261120 03:00:59.372001  1485 raft_consensus.cc:534] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 3d30c2428730481db9b0ffd279dd3440)
I20261120 03:00:59.372411  1485 raft_consensus.cc:3296] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Advancing to term 2
I20261120 03:00:59.326256  1295 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 103) took 2474 ms. Trace:
I20261120 03:00:59.372975  1295 rpcz_store.cc:276] 1120 03:00:56.852349 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:56.852455 (+   106us) service_pool.cc:224] Handling call
1120 03:00:58.919473 (+2067018us) ts_tablet_manager.cc:1197] Deleting tablet a6ae8007455d4be3b50672b9c6b0fd8f
1120 03:00:58.919513 (+    40us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:58.967571 (+ 48058us) spinlock_profiling.cc:245] Waited 45.7 ms on lock 0x7b54000f2154. stack: 00007f57a3e09023 00007f57a3e08f20 00007f57a39a4574 00007f57ad5c1964 00007f57ad5c18f9 00007f57aaf31a41 00007f57aafd26a2 00007f57ab43286c 00007f57ac09dffa 00007f57ac0d00b0 00007f57ac0b643d 00007f57ac0b63e9 00007f57ac0b6379 00007f57ac0b6341 00007f57ac0b556d 0000000000327654
1120 03:00:59.264635 (+297064us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:59.302879 (+ 38244us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:59.326198 (+ 23319us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":158656,"spinlock_wait_cycles":105083264,"tablet-delete.queue_time_us":2032859}
I20261120 03:00:59.374191   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet a6ae8007455d4be3b50672b9c6b0fd8f (table default.Table_f73e1375172b4e9999e9906c1491c0d5 [id=96692554644a476d87e6bdcaa3bd847c]) successfully deleted
I20261120 03:00:59.348645  1180 tablet_bootstrap.cc:654] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:59.338608   898 tablet_service.cc:1495] Processing CreateTablet for tablet 0dca5a2820ad4197b83a804b0310e57c (DEFAULT_TABLE table=default.Table_331ec71dbe224231bb5e1cce1470d2ea [id=f3b2bd5e97a6498ca79bf605a3ce28f6]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.375685   753 tablet_service.cc:1495] Processing CreateTablet for tablet 0dca5a2820ad4197b83a804b0310e57c (DEFAULT_TABLE table=default.Table_331ec71dbe224231bb5e1cce1470d2ea [id=f3b2bd5e97a6498ca79bf605a3ce28f6]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.376466  1295 tablet_replica.cc:333] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:59.376648   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0dca5a2820ad4197b83a804b0310e57c. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:59.385596  1735 consensus_queue.cc:1049] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:59.389503  1761 ts_tablet_manager.cc:1996] T c6a65c1a4a574961bec657ad7c9048e0 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:59.386888   898 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0dca5a2820ad4197b83a804b0310e57c. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:59.401767  1181 tablet_bootstrap.cc:654] T dc4133a8f33b4902a6308f9bdd4f91d2 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:59.402731  1295 raft_consensus.cc:2440] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Raft consensus shutting down.
W20261120 03:00:59.408495  1011 leader_election.cc:343] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: 200b234da897419a82bf5844620adf24
I20261120 03:00:59.408924  1011 leader_election.cc:304] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:59.409250   499 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet d736d65c3e0b419084b4934dad83f16a (table default.Table_04ddab7f706b43e7ba12fa247b3ca933 [id=9f571a69327a498e89f323c9ac0eee34]). Expected version 1 got 0
I20261120 03:00:59.409588  1756 raft_consensus.cc:2966] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:59.409899   499 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 0db7db25d779459ebde532f938493975 (table default.Table_faa63585fa0a40f7b9fe1e97441cdc7a [id=f3b5f7a4e5494e82a472d0025df96e0c]). Expected version 1 got 0
I20261120 03:00:59.373258  1485 raft_consensus.cc:556] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:59.411293   499 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
I20261120 03:00:59.414731   901 tablet_service.cc:1495] Processing CreateTablet for tablet bed7910ab7c140199de04e306c532969 (DEFAULT_TABLE table=default.Table_7b88ce43c2c84af2910d96587817965b [id=18d9f476562b4696aeb5d11ac7ab4d34]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.415406  1055 tablet_service.cc:1495] Processing CreateTablet for tablet bed7910ab7c140199de04e306c532969 (DEFAULT_TABLE table=default.Table_7b88ce43c2c84af2910d96587817965b [id=18d9f476562b4696aeb5d11ac7ab4d34]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.415755   901 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bed7910ab7c140199de04e306c532969. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:59.416973   756 tablet_service.cc:1495] Processing CreateTablet for tablet bed7910ab7c140199de04e306c532969 (DEFAULT_TABLE table=default.Table_7b88ce43c2c84af2910d96587817965b [id=18d9f476562b4696aeb5d11ac7ab4d34]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.417919   756 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bed7910ab7c140199de04e306c532969. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:59.422420  1485 leader_election.cc:290] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 2 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:59.423198  1295 raft_consensus.cc:2469] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:59.430023  1324 tablet_bootstrap.cc:492] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:00:59.434978  1295 ts_tablet_manager.cc:1962] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
W20261120 03:00:59.435406   710 leader_election.cc:343] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 2 election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Not found: Tablet not found: 1885397e42774a89bb347bc776284bc0
W20261120 03:00:59.442138   857 consensus_peers.cc:597] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:59.443986  1079 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1885397e42774a89bb347bc776284bc0" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 2 candidate_status { last_received { term: 1 index: 1 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:00:59.444559  1079 raft_consensus.cc:3296] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 2
I20261120 03:00:59.445430  1079 raft_consensus.cc:2062] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234 [term 2 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 2
I20261120 03:00:59.465144   709 leader_election.cc:304] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440
I20261120 03:00:59.466650  1769 raft_consensus.cc:3024] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 2 FOLLOWER]: Leader election won for term 2
I20261120 03:00:59.467149  1769 raft_consensus.cc:744] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 2 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:00:59.468000  1769 consensus_queue.cc:238] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 1.1, Last appended by leader: 1, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:59.470965   501 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 8935b23c661c4c1ca7bde7c893927502 consensus state has no leader
I20261120 03:00:59.471501   501 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:45640 (request call id 162) took 1836 ms. Trace:
I20261120 03:00:59.471873   501 rpcz_store.cc:276] 1120 03:00:57.635354 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.334822 (+1699468us) service_pool.cc:224] Handling call
1120 03:00:59.344480 (+  9658us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:00:59.390554 (+ 46074us) catalog_manager.cc:3769] Apply alter schema
1120 03:00:59.391143 (+   589us) catalog_manager.cc:3801] Acquired catalog manager lock
1120 03:00:59.391353 (+   210us) catalog_manager.cc:3972] Updating metadata on disk
1120 03:00:59.470227 (+ 78874us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:59.470252 (+    25us) write_op.cc:459] Released schema lock
1120 03:00:59.470453 (+   201us) catalog_manager.cc:4003] Committing alterations to in-memory state
1120 03:00:59.471448 (+   995us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:59.445615 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:59.446017 (+   402us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:59.446042 (+    25us) write_op.cc:440] Acquired schema lock
1120 03:00:59.446054 (+    12us) tablet.cc:710] Decoding operations
1120 03:00:59.446361 (+   307us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:59.446403 (+    42us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:59.446424 (+    21us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:00:59.446581 (+   157us) tablet.cc:758] Row locks acquired
1120 03:00:59.446593 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:00:59.446712 (+   119us) write_op.cc:270] Start()
1120 03:00:59.446796 (+    84us) write_op.cc:276] Timestamp: P: 1795143659446645 usec, L: 0
1120 03:00:59.446811 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:00:59.447302 (+   491us) log.cc:844] Serialized 405 byte log entry
1120 03:00:59.463064 (+ 15762us) op_driver.cc:466] REPLICATION: finished
1120 03:00:59.463413 (+   349us) write_op.cc:301] APPLY: starting
1120 03:00:59.463472 (+    59us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:59.463575 (+   103us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:59.463675 (+   100us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:59.463812 (+   137us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:59.463931 (+   119us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:59.463970 (+    39us) write_op.cc:312] APPLY: finished
1120 03:00:59.464167 (+   197us) log.cc:844] Serialized 23 byte log entry
1120 03:00:59.464300 (+   133us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:59.464367 (+    67us) write_op.cc:459] Released schema lock
1120 03:00:59.464397 (+    30us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"child_traces":[["op",{"apply.queue_time_us":201,"apply.run_cpu_time_us":1152,"apply.run_wall_time_us":1151,"num_ops":1,"prepare.queue_time_us":53129,"prepare.run_cpu_time_us":1968,"prepare.run_wall_time_us":1968,"raft.queue_time_us":1077,"raft.run_cpu_time_us":679,"raft.run_wall_time_us":680,"replication_time_us":16154}]]}
I20261120 03:00:59.494692  1055 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet bed7910ab7c140199de04e306c532969. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:59.495702  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 85) took 5226 ms. Trace:
I20261120 03:00:59.496019  1392 rpcz_store.cc:276] 1120 03:00:54.269388 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.269536 (+   148us) service_pool.cc:224] Handling call
1120 03:00:58.893704 (+4624168us) ts_tablet_manager.cc:1197] Deleting tablet df61317717d3401eb74cd93aa336c4dc
1120 03:00:58.893763 (+    59us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:59.147904 (+254141us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:59.359982 (+212078us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:59.495644 (+135662us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":4623555}
I20261120 03:00:59.497103  1392 tablet_replica.cc:333] T a08aa5ddcf1a4061a70275a8eecb1856 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:59.497828  1181 tablet_bootstrap.cc:492] T dc4133a8f33b4902a6308f9bdd4f91d2 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:59.498289  1181 ts_tablet_manager.cc:1449] T dc4133a8f33b4902a6308f9bdd4f91d2 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.209s	user 0.007s	sys 0.003s
I20261120 03:00:59.497877  1392 raft_consensus.cc:2440] T a08aa5ddcf1a4061a70275a8eecb1856 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:59.499334  1392 raft_consensus.cc:2469] T a08aa5ddcf1a4061a70275a8eecb1856 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:59.501386  1392 ts_tablet_manager.cc:1962] T a08aa5ddcf1a4061a70275a8eecb1856 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
W20261120 03:00:59.506043   710 consensus_peers.cc:597] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Couldn't send request to peer 3d30c2428730481db9b0ffd279dd3440. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1885397e42774a89bb347bc776284bc0. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:59.511821   903 tablet_service.cc:1495] Processing CreateTablet for tablet 8935b23c661c4c1ca7bde7c893927502 (DEFAULT_TABLE table=default.Table_92f6cc71382f4deba2472c49ae8f6a36 [id=51df6e371d3441578e57ea5597ee81a6]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.512845   903 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8935b23c661c4c1ca7bde7c893927502. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:59.512773  1079 raft_consensus.cc:1338] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234 [term 2 FOLLOWER]: Refusing update from remote peer 5beeefe8bef54caeac463d05d8c30fa6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 2 index: 2. (index mismatch)
I20261120 03:00:59.500768  1181 raft_consensus.cc:366] T dc4133a8f33b4902a6308f9bdd4f91d2 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:59.516281  1181 raft_consensus.cc:392] T dc4133a8f33b4902a6308f9bdd4f91d2 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:59.516559  1181 raft_consensus.cc:787] T dc4133a8f33b4902a6308f9bdd4f91d2 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
W20261120 03:00:59.514147   709 consensus_peers.cc:597] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:00:59.517735  1769 consensus_queue.cc:1049] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:00:59.521942  1768 raft_consensus.cc:534] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:59.522729  1768 raft_consensus.cc:556] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:59.524679  1768 leader_election.cc:290] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:59.498661   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet df61317717d3401eb74cd93aa336c4dc (table default.Table_e4c5ff8f28ae4efca67257942a4fcb77 [id=b30488f6d14b4c9ab1566f2376be602a]) successfully deleted
W20261120 03:00:59.503484   488 catalog_manager.cc:4863] Async tablet task AlterTable RPC for tablet 8935b23c661c4c1ca7bde7c893927502 (table default.Table_92f6cc71382f4deba2472c49ae8f6a36 [id=51df6e371d3441578e57ea5597ee81a6], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 8935b23c661c4c1ca7bde7c893927502 consensus state has no leader
I20261120 03:00:59.525722  1079 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "73c993944b0a4977b55d40ad63bb69e3" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:59.527359   752 tablet_service.cc:1495] Processing CreateTablet for tablet 0c36259ff308455d986243c6b07d81a6 (DEFAULT_TABLE table=default.Table_b2cdcfa34e9949afa95603981f35136f [id=61ed1ffc198f476f9b3f97d44b42694a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.527827  1181 consensus_queue.cc:261] T dc4133a8f33b4902a6308f9bdd4f91d2 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:59.529471  1045 tablet_service.cc:1495] Processing CreateTablet for tablet 0c36259ff308455d986243c6b07d81a6 (DEFAULT_TABLE table=default.Table_b2cdcfa34e9949afa95603981f35136f [id=61ed1ffc198f476f9b3f97d44b42694a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.530483   501 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_150226bfe577411d88c38a63b1af73e5"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:00:59.530911  1045 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0c36259ff308455d986243c6b07d81a6. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:59.531107   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "73c993944b0a4977b55d40ad63bb69e3" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:00:59.532501   710 leader_election.cc:343] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:59.509783  1180 tablet_bootstrap.cc:492] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:00:59.533295  1180 ts_tablet_manager.cc:1449] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.214s	user 0.000s	sys 0.016s
I20261120 03:00:59.535672  1180 raft_consensus.cc:366] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
W20261120 03:00:59.539017   709 leader_election.cc:343] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:59.539395   709 leader_election.cc:304] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:00:59.540633  1769 raft_consensus.cc:2966] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:59.542847  1181 ts_tablet_manager.cc:1480] T dc4133a8f33b4902a6308f9bdd4f91d2 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.044s	user 0.005s	sys 0.001s
I20261120 03:00:59.543728  1181 tablet_bootstrap.cc:492] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:59.555570  1761 tablet_replica.cc:333] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:00:59.559692  1761 raft_consensus.cc:2440] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:59.560150  1761 raft_consensus.cc:2469] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:59.561982  1761 ts_tablet_manager.cc:1962] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:59.566651   752 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0c36259ff308455d986243c6b07d81a6. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:59.584091  1181 tablet_bootstrap.cc:654] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:59.585041  1324 tablet_bootstrap.cc:654] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
W20261120 03:00:59.597218   501 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_150226bfe577411d88c38a63b1af73e5 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:59.518441   497 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
I20261120 03:00:59.559077   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet c6a65c1a4a574961bec657ad7c9048e0 (table default.Table_dd2f7617ff0e401b9b862d5991110dd5 [id=3955f41b6d4d41bf8ba2c81e4ac7f4a8]) successfully deleted
I20261120 03:00:59.606964   497 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 1faba22e8e38418383b86e42e904a761 (table default.Table_47101d55d8f6485c816a02a54fda67a8 [id=252d229ff1334cd99ec99e5c921031bb]). Expected version 1 got 0
I20261120 03:00:59.610369   756 tablet_service.cc:1495] Processing CreateTablet for tablet 8f1fda857e6c400e80d3d177a3d748ff (DEFAULT_TABLE table=default.Table_f73e1375172b4e9999e9906c1491c0d5 [id=96692554644a476d87e6bdcaa3bd847c]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.611531   756 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8f1fda857e6c400e80d3d177a3d748ff. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:59.614969   753 tablet_service.cc:1495] Processing CreateTablet for tablet 8935b23c661c4c1ca7bde7c893927502 (DEFAULT_TABLE table=default.Table_92f6cc71382f4deba2472c49ae8f6a36 [id=51df6e371d3441578e57ea5597ee81a6]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.615991   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8935b23c661c4c1ca7bde7c893927502. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:59.616671   497 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 0db7db25d779459ebde532f938493975 (table default.Table_faa63585fa0a40f7b9fe1e97441cdc7a [id=f3b5f7a4e5494e82a472d0025df96e0c]). Expected version 1 got 0
I20261120 03:00:59.617372   497 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet d736d65c3e0b419084b4934dad83f16a (table default.Table_04ddab7f706b43e7ba12fa247b3ca933 [id=9f571a69327a498e89f323c9ac0eee34]). Expected version 1 got 0
I20261120 03:00:59.619920  1181 tablet_bootstrap.cc:492] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:59.620329  1181 ts_tablet_manager.cc:1449] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.077s	user 0.008s	sys 0.003s
I20261120 03:00:59.621358   497 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 0179a59749a14168a840e9d4e610be1f (table default.Table_cc56bf0ad8344f1a94d5da6e4d77e8c0 [id=b9034a083c7e4133a746b336335d3e42]). Expected version 1 got 0
I20261120 03:00:59.622673  1181 raft_consensus.cc:366] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:59.623272  1181 raft_consensus.cc:392] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:59.623512  1181 raft_consensus.cc:787] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:59.624364  1756 raft_consensus.cc:534] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:59.624289  1181 consensus_queue.cc:261] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:59.624794  1756 raft_consensus.cc:556] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:59.626775  1756 leader_election.cc:290] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:00:59.538211   901 tablet_service.cc:1495] Processing CreateTablet for tablet 0c36259ff308455d986243c6b07d81a6 (DEFAULT_TABLE table=default.Table_b2cdcfa34e9949afa95603981f35136f [id=61ed1ffc198f476f9b3f97d44b42694a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.630903   901 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 0c36259ff308455d986243c6b07d81a6. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:59.569837  1295 ts_tablet_manager.cc:1975] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:59.633929   774 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3d89fb1d7fa14ae8ac252c68338ed4d6" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:59.634244  1295 log.cc:1199] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/014bd9c83be34c31b90fe4bcf31f1649
I20261120 03:00:59.634914  1295 ts_tablet_manager.cc:1996] T 014bd9c83be34c31b90fe4bcf31f1649 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:00:59.616652  1055 tablet_service.cc:1495] Processing CreateTablet for tablet 8935b23c661c4c1ca7bde7c893927502 (DEFAULT_TABLE table=default.Table_92f6cc71382f4deba2472c49ae8f6a36 [id=51df6e371d3441578e57ea5597ee81a6]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.636776  1295 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 104) took 2783 ms. Trace:
I20261120 03:00:59.637336  1055 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8935b23c661c4c1ca7bde7c893927502. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:59.637084  1295 rpcz_store.cc:276] 1120 03:00:56.852885 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:56.853013 (+   128us) service_pool.cc:224] Handling call
1120 03:00:59.376324 (+2523311us) ts_tablet_manager.cc:1197] Deleting tablet 014bd9c83be34c31b90fe4bcf31f1649
1120 03:00:59.376363 (+    39us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:59.519883 (+143520us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:59.569721 (+ 49838us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:59.636722 (+ 67001us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":19826,"tablet-delete.queue_time_us":2489552}
I20261120 03:00:59.593360   895 tablet_service.cc:1495] Processing CreateTablet for tablet 8f1fda857e6c400e80d3d177a3d748ff (DEFAULT_TABLE table=default.Table_f73e1375172b4e9999e9906c1491c0d5 [id=96692554644a476d87e6bdcaa3bd847c]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.638764   895 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8f1fda857e6c400e80d3d177a3d748ff. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:59.629221   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3d89fb1d7fa14ae8ac252c68338ed4d6" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:00:59.645992  1012 leader_election.cc:304] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:00:59.647303  1181 ts_tablet_manager.cc:1480] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.027s	user 0.007s	sys 0.000s
I20261120 03:00:59.647320  1756 raft_consensus.cc:2966] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:00:59.648325  1181 tablet_bootstrap.cc:492] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:59.654122  1181 tablet_bootstrap.cc:654] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:59.664600  1181 tablet_bootstrap.cc:492] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:00:59.664999  1181 ts_tablet_manager.cc:1449] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.017s	user 0.008s	sys 0.007s
I20261120 03:00:59.665254  1295 tablet_replica.cc:333] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:00:59.666047  1295 raft_consensus.cc:2440] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:00:59.666646  1295 raft_consensus.cc:2469] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:59.667237  1181 raft_consensus.cc:366] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:59.667814  1181 raft_consensus.cc:392] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:59.668051  1181 raft_consensus.cc:787] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:00:59.668593  1181 consensus_queue.cc:261] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:59.669692  1392 ts_tablet_manager.cc:1975] T a08aa5ddcf1a4061a70275a8eecb1856 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:59.670336  1392 log.cc:1199] T a08aa5ddcf1a4061a70275a8eecb1856 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/a08aa5ddcf1a4061a70275a8eecb1856
I20261120 03:00:59.670511  1181 ts_tablet_manager.cc:1480] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.005s	user 0.006s	sys 0.001s
I20261120 03:00:59.671492  1392 ts_tablet_manager.cc:1996] T a08aa5ddcf1a4061a70275a8eecb1856 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:00:59.635878  1739 raft_consensus.cc:366] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:59.675197  1739 raft_consensus.cc:787] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2599fd9eef7d4ce990c40e2356a37c3f, State: Initialized, Role: FOLLOWER
I20261120 03:00:59.677104  1739 consensus_queue.cc:261] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 9, Last appended: 3.9, Last appended by leader: 9, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:00:59.696637  1180 raft_consensus.cc:392] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:59.696947  1180 raft_consensus.cc:787] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:00:59.697584  1180 consensus_queue.cc:261] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:59.699539  1180 ts_tablet_manager.cc:1480] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.166s	user 0.004s	sys 0.001s
I20261120 03:00:59.700492  1180 tablet_bootstrap.cc:492] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:00:59.704197   970 heartbeater.cc:345] Connected to a master server at 127.0.96.124:38133
I20261120 03:00:59.704303  1045 tablet_service.cc:1495] Processing CreateTablet for tablet 8f1fda857e6c400e80d3d177a3d748ff (DEFAULT_TABLE table=default.Table_f73e1375172b4e9999e9906c1491c0d5 [id=96692554644a476d87e6bdcaa3bd847c]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.709698  1051 tablet_service.cc:1495] Processing CreateTablet for tablet 4d79fc4c8f1c4a5f83c0dc94337c857e (DEFAULT_TABLE table=default.Table_650dc67e0ac74c149a3f64d456767938 [id=cfcd5fea97fb4ab3baff99eb28cd9476]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.713645  1295 ts_tablet_manager.cc:1962] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:59.716825  1181 tablet_bootstrap.cc:492] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:00:59.719022  1324 tablet_bootstrap.cc:492] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:00:59.719458  1324 ts_tablet_manager.cc:1449] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.290s	user 0.003s	sys 0.008s
I20261120 03:00:59.721055   753 tablet_service.cc:1495] Processing CreateTablet for tablet 4d79fc4c8f1c4a5f83c0dc94337c857e (DEFAULT_TABLE table=default.Table_650dc67e0ac74c149a3f64d456767938 [id=cfcd5fea97fb4ab3baff99eb28cd9476]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.721678  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 86) took 5402 ms. Trace:
I20261120 03:00:59.721982  1392 rpcz_store.cc:276] 1120 03:00:54.319758 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.321914 (+  2156us) service_pool.cc:224] Handling call
1120 03:00:59.496944 (+5175030us) ts_tablet_manager.cc:1197] Deleting tablet a08aa5ddcf1a4061a70275a8eecb1856
1120 03:00:59.497004 (+    60us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:59.549816 (+ 52812us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:59.669498 (+119682us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:59.721625 (+ 52127us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":5174435}
I20261120 03:00:59.722982  1392 tablet_replica.cc:333] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:00:59.723665  1392 raft_consensus.cc:2440] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:00:59.724484  1392 raft_consensus.cc:2469] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:00:59.716591  1782 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:00:59.727437  1782 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:00:59.728433  1392 ts_tablet_manager.cc:1962] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:59.730379  1045 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8f1fda857e6c400e80d3d177a3d748ff. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:59.732240  1181 tablet_bootstrap.cc:654] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:00:59.739722  1051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4d79fc4c8f1c4a5f83c0dc94337c857e. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:59.643991   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 014bd9c83be34c31b90fe4bcf31f1649 (table default.Table_cfa8989fb1134d908ab28bd6ebb80563 [id=16bb3f7e35c54a8c936b4af5a66a7f17]) successfully deleted
W20261120 03:00:59.764036   487 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 120058us
W20261120 03:00:59.765357   487 catalog_manager.cc:5061] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet b549a06d15db449b851fe9549a963416 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet b549a06d15db449b851fe9549a963416 already in progress: creating tablet
W20261120 03:00:59.765959   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet ce612f9a121b4b18908724d2726e156f because the tablet was not found. No further retry: Not found: Tablet not found: ce612f9a121b4b18908724d2726e156f
W20261120 03:00:59.766512   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 020be7f7cda0467aab56f657cbe8d0ad because the tablet was not found. No further retry: Not found: Tablet not found: 020be7f7cda0467aab56f657cbe8d0ad
W20261120 03:00:59.767045   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 68179488b9bc4e03b65a9f1d434c95b5 because the tablet was not found. No further retry: Not found: Tablet not found: 68179488b9bc4e03b65a9f1d434c95b5
W20261120 03:00:59.767576   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 1885397e42774a89bb347bc776284bc0 because the tablet was not found. No further retry: Not found: Tablet not found: 1885397e42774a89bb347bc776284bc0
W20261120 03:00:59.768085   487 catalog_manager.cc:5061] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet c6a65c1a4a574961bec657ad7c9048e0 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet c6a65c1a4a574961bec657ad7c9048e0 already in progress: creating tablet
I20261120 03:00:59.726591   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet a08aa5ddcf1a4061a70275a8eecb1856 (table default.Table_3c79e29085ad4875874397eaefcab250 [id=47e54c07e54d40adb2ac6907cb03bb68]) successfully deleted
I20261120 03:00:59.770792   898 tablet_service.cc:1495] Processing CreateTablet for tablet 4d79fc4c8f1c4a5f83c0dc94337c857e (DEFAULT_TABLE table=default.Table_650dc67e0ac74c149a3f64d456767938 [id=cfcd5fea97fb4ab3baff99eb28cd9476]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:00:59.771893   898 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4d79fc4c8f1c4a5f83c0dc94337c857e. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:00:59.779385   499 catalog_manager.cc:6409] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: default.Table_0879bd445c234e4faefff97cd088035d [id=53fb1849755c4debad2dbfbf37271fbd] alter complete (version 1)
I20261120 03:00:59.782536  1739 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: configured and running, proceeding with master startup.
I20261120 03:00:59.783072   501 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:45640 (request call id 163) took 2141 ms. Trace:
I20261120 03:00:59.783576   501 rpcz_store.cc:276] 1120 03:00:57.641878 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.530107 (+1888229us) service_pool.cc:224] Handling call
1120 03:00:59.597651 (+ 67544us) catalog_manager.cc:2145] Acquired catalog manager lock
1120 03:00:59.599043 (+  1392us) catalog_manager.cc:2191] Created new table and tablet info
1120 03:00:59.782384 (+183341us) catalog_manager.cc:2251] Wrote table and tablets to system table
1120 03:00:59.782659 (+   275us) catalog_manager.cc:2285] Inserted table and tablets into CatalogManager maps
1120 03:00:59.783015 (+   356us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:59.720108 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:59.720498 (+   390us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:59.720522 (+    24us) write_op.cc:440] Acquired schema lock
1120 03:00:59.720533 (+    11us) tablet.cc:710] Decoding operations
1120 03:00:59.720831 (+   298us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:59.720871 (+    40us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:59.720892 (+    21us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:00:59.721081 (+   189us) tablet.cc:758] Row locks acquired
1120 03:00:59.721094 (+    13us) write_op.cc:260] PREPARE: finished
1120 03:00:59.721209 (+   115us) write_op.cc:270] Start()
1120 03:00:59.721293 (+    84us) write_op.cc:276] Timestamp: P: 1795143659721170 usec, L: 0
1120 03:00:59.721308 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:00:59.721814 (+   506us) log.cc:844] Serialized 433 byte log entry
1120 03:00:59.771351 (+ 49537us) op_driver.cc:466] REPLICATION: finished
1120 03:00:59.771791 (+   440us) write_op.cc:301] APPLY: starting
1120 03:00:59.771850 (+    59us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:59.771954 (+   104us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:59.772054 (+   100us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:59.772441 (+   387us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:59.772585 (+   144us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
1120 03:00:59.772626 (+    41us) write_op.cc:312] APPLY: finished
1120 03:00:59.772802 (+   176us) log.cc:844] Serialized 29 byte log entry
1120 03:00:59.772936 (+   134us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:59.773004 (+    68us) write_op.cc:459] Released schema lock
1120 03:00:59.773034 (+    30us) write_op.cc:346] FINISH: Updating metrics
1120 03:00:59.773242 (+   208us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:59.773265 (+    23us) write_op.cc:459] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":288,"apply.run_cpu_time_us":1427,"apply.run_wall_time_us":1427,"num_ops":2,"prepare.queue_time_us":119891,"prepare.run_cpu_time_us":1966,"prepare.run_wall_time_us":1967,"raft.queue_time_us":1125,"raft.run_cpu_time_us":682,"raft.run_wall_time_us":682,"replication_time_us":49948}]]}
W20261120 03:00:59.809293   500 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.67:53803 (request call id 18) took 2195 ms (client timeout 1000 ms). Trace:
I20261120 03:00:59.817656  1392 ts_tablet_manager.cc:1975] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:00:59.825462  1392 log.cc:1199] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/ce612f9a121b4b18908724d2726e156f
I20261120 03:00:59.826232  1392 ts_tablet_manager.cc:1996] T ce612f9a121b4b18908724d2726e156f P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
W20261120 03:00:59.838865   499 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.67:53803 (request call id 19) took 1245 ms (client timeout 1000 ms). Trace:
I20261120 03:00:59.819097   493 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:45640 (request call id 155) took 4522 ms. Trace:
I20261120 03:00:59.841949   493 rpcz_store.cc:276] 1120 03:00:55.297168 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:56.832746 (+1535578us) service_pool.cc:224] Handling call
1120 03:00:57.042629 (+209883us) spinlock_profiling.cc:245] Waited 209 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07f626 00007f155ac007f4 00007f155ac00782 00007f155ac006b1 00007f155ac0062c 00007f155abff8e2 00007f1558e990bc 00007f1558e984f6
1120 03:00:57.043326 (+   697us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:00:57.044025 (+   699us) catalog_manager.cc:3769] Apply alter schema
1120 03:00:57.061227 (+ 17202us) catalog_manager.cc:3801] Acquired catalog manager lock
1120 03:00:57.061481 (+   254us) catalog_manager.cc:3972] Updating metadata on disk
1120 03:00:57.905238 (+843757us) catalog_manager.cc:4003] Committing alterations to in-memory state
1120 03:00:59.819040 (+1913802us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:57.508472 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:57.508907 (+   435us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:57.508931 (+    24us) write_op.cc:440] Acquired schema lock
1120 03:00:57.508944 (+    13us) tablet.cc:710] Decoding operations
1120 03:00:57.509201 (+   257us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:57.509239 (+    38us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:57.509260 (+    21us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:00:57.509403 (+   143us) tablet.cc:758] Row locks acquired
1120 03:00:57.509415 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:00:57.509523 (+   108us) write_op.cc:270] Start()
1120 03:00:57.509612 (+    89us) write_op.cc:276] Timestamp: P: 1795143657509446 usec, L: 0
1120 03:00:57.509627 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:00:57.510137 (+   510us) log.cc:844] Serialized 405 byte log entry
1120 03:00:57.902481 (+392344us) op_driver.cc:466] REPLICATION: finished
1120 03:00:57.902689 (+   208us) write_op.cc:301] APPLY: starting
1120 03:00:57.902732 (+    43us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:57.902814 (+    82us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:57.902903 (+    89us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:57.903084 (+   181us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:57.903223 (+   139us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:57.903268 (+    45us) write_op.cc:312] APPLY: finished
1120 03:00:57.903447 (+   179us) log.cc:844] Serialized 23 byte log entry
1120 03:00:57.903582 (+   135us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:57.903636 (+    54us) write_op.cc:459] Released schema lock
1120 03:00:57.903665 (+    29us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"mutex_wait_us":23946,"spinlock_wait_cycles":480926464,"child_traces":[["op",{"apply.queue_time_us":73,"apply.run_cpu_time_us":1176,"apply.run_wall_time_us":1171,"num_ops":1,"prepare.queue_time_us":442577,"prepare.run_cpu_time_us":1919,"prepare.run_wall_time_us":147257,"replication_time_us":392775}]]}
W20261120 03:00:59.846482   499 rpcz_store.cc:269] 1120 03:00:58.593216 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:58.976349 (+383133us) service_pool.cc:224] Handling call
1120 03:00:59.588043 (+611694us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:59.588070 (+    27us) write_op.cc:459] Released schema lock
1120 03:00:59.838797 (+250727us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:59.531733 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:59.532137 (+   404us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:59.532162 (+    25us) write_op.cc:440] Acquired schema lock
1120 03:00:59.532175 (+    13us) tablet.cc:710] Decoding operations
1120 03:00:59.532442 (+   267us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:59.532482 (+    40us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:59.532504 (+    22us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:00:59.532672 (+   168us) tablet.cc:758] Row locks acquired
1120 03:00:59.532686 (+    14us) write_op.cc:260] PREPARE: finished
1120 03:00:59.532796 (+   110us) write_op.cc:270] Start()
1120 03:00:59.532873 (+    77us) write_op.cc:276] Timestamp: P: 1795143659532725 usec, L: 0
1120 03:00:59.532887 (+    14us) op_driver.cc:350] REPLICATION: starting
1120 03:00:59.533361 (+   474us) log.cc:844] Serialized 356 byte log entry
1120 03:00:59.542973 (+  9612us) op_driver.cc:466] REPLICATION: finished
1120 03:00:59.543305 (+   332us) write_op.cc:301] APPLY: starting
1120 03:00:59.543360 (+    55us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:59.543453 (+    93us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:59.543546 (+    93us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:59.543730 (+   184us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:59.543851 (+   121us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:59.543892 (+    41us) write_op.cc:312] APPLY: finished
1120 03:00:59.544088 (+   196us) log.cc:844] Serialized 23 byte log entry
1120 03:00:59.544218 (+   130us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:59.544282 (+    64us) write_op.cc:459] Released schema lock
1120 03:00:59.544310 (+    28us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"reported_tablets":23,"child_traces":[["op",{"apply.queue_time_us":186,"apply.run_cpu_time_us":1165,"apply.run_wall_time_us":1165,"num_ops":1,"prepare.queue_time_us":10454,"prepare.run_cpu_time_us":1916,"prepare.run_wall_time_us":1916,"raft.queue_time_us":1151,"raft.run_cpu_time_us":653,"raft.run_wall_time_us":653,"replication_time_us":9993}]]}
I20261120 03:00:59.853901   895 tablet_service.cc:1542] Processing DeleteTablet for tablet 88b9e9692a874ae993dde4b9e2586a70 with delete_type TABLET_DATA_DELETED (replaced by tablet 9e42a416be884341994d2d5395286798) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:59.856985   901 tablet_service.cc:1542] Processing DeleteTablet for tablet ce612f9a121b4b18908724d2726e156f with delete_type TABLET_DATA_DELETED (replaced by tablet 5da3d2724dd34aed88e6b08983188004) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:59.857563   903 tablet_service.cc:1542] Processing DeleteTablet for tablet 5c5c0ac869a1419fad9f61ec647027ee with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:53 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:59.858211   896 tablet_service.cc:1542] Processing DeleteTablet for tablet 1885397e42774a89bb347bc776284bc0 with delete_type TABLET_DATA_DELETED (replaced by tablet b549a06d15db449b851fe9549a963416) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:59.859831   900 tablet_service.cc:1542] Processing DeleteTablet for tablet 014bd9c83be34c31b90fe4bcf31f1649 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:54 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:59.861917   902 tablet_service.cc:1542] Processing DeleteTablet for tablet a6ae8007455d4be3b50672b9c6b0fd8f with delete_type TABLET_DATA_DELETED (replaced by tablet 8f1fda857e6c400e80d3d177a3d748ff) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:59.862569   895 tablet_service.cc:1542] Processing DeleteTablet for tablet 020be7f7cda0467aab56f657cbe8d0ad with delete_type TABLET_DATA_DELETED (replaced by tablet 600a18b7e5104a78935d906b8adfde6f) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:59.863154   898 tablet_service.cc:1542] Processing DeleteTablet for tablet 68179488b9bc4e03b65a9f1d434c95b5 with delete_type TABLET_DATA_DELETED (replaced by tablet 4d79fc4c8f1c4a5f83c0dc94337c857e) from {username='slave'} at 127.0.0.1:44896
I20261120 03:00:59.879990   755 tablet_service.cc:1542] Processing DeleteTablet for tablet 020be7f7cda0467aab56f657cbe8d0ad with delete_type TABLET_DATA_DELETED (replaced by tablet 600a18b7e5104a78935d906b8adfde6f) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:59.880721   749 tablet_service.cc:1542] Processing DeleteTablet for tablet 5c5c0ac869a1419fad9f61ec647027ee with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:53 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:59.881338   755 tablet_service.cc:1542] Processing DeleteTablet for tablet a08aa5ddcf1a4061a70275a8eecb1856 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:53 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:59.880721   757 tablet_service.cc:1542] Processing DeleteTablet for tablet a6ae8007455d4be3b50672b9c6b0fd8f with delete_type TABLET_DATA_DELETED (replaced by tablet 8f1fda857e6c400e80d3d177a3d748ff) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:59.882543   754 tablet_service.cc:1542] Processing DeleteTablet for tablet 88b9e9692a874ae993dde4b9e2586a70 with delete_type TABLET_DATA_DELETED (replaced by tablet 9e42a416be884341994d2d5395286798) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:59.883715   751 tablet_service.cc:1542] Processing DeleteTablet for tablet 68179488b9bc4e03b65a9f1d434c95b5 with delete_type TABLET_DATA_DELETED (replaced by tablet 4d79fc4c8f1c4a5f83c0dc94337c857e) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:59.884430   750 tablet_service.cc:1542] Processing DeleteTablet for tablet 1885397e42774a89bb347bc776284bc0 with delete_type TABLET_DATA_DELETED (replaced by tablet b549a06d15db449b851fe9549a963416) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:59.885397  1324 raft_consensus.cc:366] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:59.886484   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4d79fc4c8f1c4a5f83c0dc94337c857e. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:00:59.813481   494 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.65:40295 (request call id 31) took 4698 ms (client timeout 1000 ms). Trace:
W20261120 03:00:59.894321   494 rpcz_store.cc:269] 1120 03:00:55.115226 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:55.372137 (+256911us) service_pool.cc:224] Handling call
1120 03:00:55.586287 (+214150us) spinlock_profiling.cc:245] Waited 202 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07d9ce 00007f155abf0174 00007f155abf0102 00007f155abf0031 00007f155abeffac 00007f155abef262 00007f1558e990bc 00007f1558e984f6
1120 03:00:58.129737 (+2543450us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:58.129764 (+    27us) write_op.cc:459] Released schema lock
1120 03:00:59.813420 (+1683656us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:58.109931 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:58.110346 (+   415us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:58.110369 (+    23us) write_op.cc:440] Acquired schema lock
1120 03:00:58.110381 (+    12us) tablet.cc:710] Decoding operations
1120 03:00:58.110683 (+   302us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:58.110724 (+    41us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:58.110744 (+    20us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:00:58.111003 (+   259us) tablet.cc:758] Row locks acquired
1120 03:00:58.111014 (+    11us) write_op.cc:260] PREPARE: finished
1120 03:00:58.111135 (+   121us) write_op.cc:270] Start()
1120 03:00:58.111214 (+    79us) write_op.cc:276] Timestamp: P: 1795143658111042 usec, L: 0
1120 03:00:58.111228 (+    14us) op_driver.cc:350] REPLICATION: starting
1120 03:00:58.111684 (+   456us) log.cc:844] Serialized 901 byte log entry
1120 03:00:58.119413 (+  7729us) op_driver.cc:466] REPLICATION: finished
1120 03:00:58.119786 (+   373us) write_op.cc:301] APPLY: starting
1120 03:00:58.119841 (+    55us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:58.119950 (+   109us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:58.120038 (+    88us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:58.120622 (+   584us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:58.120746 (+   124us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
1120 03:00:58.120790 (+    44us) write_op.cc:312] APPLY: finished
1120 03:00:58.120951 (+   161us) log.cc:844] Serialized 29 byte log entry
1120 03:00:58.121081 (+   130us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:58.121161 (+    80us) write_op.cc:459] Released schema lock
1120 03:00:58.121191 (+    30us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"reported_tablets":16,"spinlock_wait_cycles":464663936,"child_traces":[["op",{"apply.queue_time_us":241,"apply.run_cpu_time_us":1562,"apply.run_wall_time_us":1561,"num_ops":2,"prepare.queue_time_us":105,"prepare.run_cpu_time_us":2052,"prepare.run_wall_time_us":2054,"raft.queue_time_us":2523,"raft.run_cpu_time_us":681,"raft.run_wall_time_us":680,"replication_time_us":8096}]]}
W20261120 03:00:59.817989   498 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.66:36141 (request call id 26) took 2961 ms (client timeout 1000 ms). Trace:
W20261120 03:00:59.898439   498 rpcz_store.cc:269] 1120 03:00:56.856408 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:57.218420 (+362012us) service_pool.cc:224] Handling call
1120 03:00:57.485399 (+266979us) spinlock_profiling.cc:245] Waited 263 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07d9ce 00007f155abf0174 00007f155abf0102 00007f155abf0031 00007f155abeffac 00007f155abef262 00007f1558e990bc 00007f1558e984f6
1120 03:00:59.817927 (+2332528us) inbound_call.cc:177] Queueing success response
Metrics: {"reported_tablets":19,"spinlock_wait_cycles":603911040}
I20261120 03:00:59.900039   498 master_service.cc:965] ReplaceTablet: received request to replace tablet 68179488b9bc4e03b65a9f1d434c95b5 from {username='slave'} at 127.0.0.1:45640
W20261120 03:00:59.851186   495 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.65:40295 (request call id 32) took 3389 ms (client timeout 1000 ms). Trace:
W20261120 03:00:59.900590   495 rpcz_store.cc:269] 1120 03:00:56.461832 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:56.754095 (+292263us) service_pool.cc:224] Handling call
1120 03:00:57.042155 (+288060us) spinlock_profiling.cc:245] Waited 281 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07d9ce 00007f155abf0174 00007f155abf0102 00007f155abf0031 00007f155abeffac 00007f155abef262 00007f1558e990bc 00007f1558e984f6
1120 03:00:59.851120 (+2808965us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:58.629989 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:58.630445 (+   456us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:58.630470 (+    25us) write_op.cc:440] Acquired schema lock
1120 03:00:58.630483 (+    13us) tablet.cc:710] Decoding operations
1120 03:00:58.630747 (+   264us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:58.630786 (+    39us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:58.630808 (+    22us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:00:58.630968 (+   160us) tablet.cc:758] Row locks acquired
1120 03:00:58.630979 (+    11us) write_op.cc:260] PREPARE: finished
1120 03:00:58.631088 (+   109us) write_op.cc:270] Start()
1120 03:00:58.631172 (+    84us) write_op.cc:276] Timestamp: P: 1795143658630996 usec, L: 0
1120 03:00:58.631188 (+    16us) op_driver.cc:350] REPLICATION: starting
1120 03:00:58.632578 (+  1390us) log.cc:844] Serialized 527 byte log entry
1120 03:00:58.641799 (+  9221us) op_driver.cc:466] REPLICATION: finished
1120 03:00:58.642171 (+   372us) write_op.cc:301] APPLY: starting
1120 03:00:58.642222 (+    51us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:58.642314 (+    92us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:58.642409 (+    95us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:58.642571 (+   162us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:58.642688 (+   117us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:58.642728 (+    40us) write_op.cc:312] APPLY: finished
1120 03:00:58.642892 (+   164us) log.cc:844] Serialized 23 byte log entry
1120 03:00:58.643026 (+   134us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:58.643089 (+    63us) write_op.cc:459] Released schema lock
1120 03:00:58.643123 (+    34us) write_op.cc:346] FINISH: Updating metrics
1120 03:00:58.643347 (+   224us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:58.643371 (+    24us) write_op.cc:459] Released schema lock
Metrics: {"reported_tablets":22,"spinlock_wait_cycles":645515008,"child_traces":[["op",{"apply.queue_time_us":199,"apply.run_cpu_time_us":1154,"apply.run_wall_time_us":1154,"num_ops":1,"prepare.queue_time_us":102,"prepare.run_cpu_time_us":2003,"prepare.run_wall_time_us":2872,"raft.queue_time_us":1095,"raft.run_cpu_time_us":715,"raft.run_wall_time_us":724,"replication_time_us":10518}]]}
W20261120 03:00:59.841549   502 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.66:36141 (request call id 28) took 882 ms (client timeout 1000 ms). Trace:
W20261120 03:00:59.904598   710 consensus_peers.cc:597] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Couldn't send request to peer 3d30c2428730481db9b0ffd279dd3440. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
W20261120 03:00:59.903398   502 rpcz_store.cc:269] 1120 03:00:58.958771 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.027308 (+ 68537us) service_pool.cc:224] Handling call
1120 03:00:59.373559 (+346251us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:59.389810 (+ 16251us) write_op.cc:459] Released schema lock
1120 03:00:59.841488 (+451678us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:59.345245 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:59.345656 (+   411us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:59.345681 (+    25us) write_op.cc:440] Acquired schema lock
1120 03:00:59.345694 (+    13us) tablet.cc:710] Decoding operations
1120 03:00:59.345997 (+   303us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:59.346036 (+    39us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:59.346057 (+    21us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:00:59.346345 (+   288us) tablet.cc:758] Row locks acquired
1120 03:00:59.346359 (+    14us) write_op.cc:260] PREPARE: finished
1120 03:00:59.346469 (+   110us) write_op.cc:270] Start()
1120 03:00:59.346544 (+    75us) write_op.cc:276] Timestamp: P: 1795143659346417 usec, L: 0
1120 03:00:59.346571 (+    27us) op_driver.cc:350] REPLICATION: starting
1120 03:00:59.347034 (+   463us) log.cc:844] Serialized 901 byte log entry
1120 03:00:59.357321 (+ 10287us) op_driver.cc:466] REPLICATION: finished
1120 03:00:59.357665 (+   344us) write_op.cc:301] APPLY: starting
1120 03:00:59.357714 (+    49us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:59.357818 (+   104us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:59.357909 (+    91us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:59.358184 (+   275us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:59.358314 (+   130us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
1120 03:00:59.358356 (+    42us) write_op.cc:312] APPLY: finished
1120 03:00:59.358530 (+   174us) log.cc:844] Serialized 29 byte log entry
1120 03:00:59.358672 (+   142us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:59.358738 (+    66us) write_op.cc:459] Released schema lock
1120 03:00:59.358769 (+    31us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"mutex_wait_us":40784,"reported_tablets":17,"child_traces":[["op",{"apply.queue_time_us":199,"apply.run_cpu_time_us":1273,"apply.run_wall_time_us":1273,"num_ops":2,"prepare.queue_time_us":2111,"prepare.run_cpu_time_us":2061,"prepare.run_wall_time_us":2061,"raft.queue_time_us":1130,"raft.run_cpu_time_us":728,"raft.run_wall_time_us":727,"replication_time_us":10657}]]}
I20261120 03:00:59.907109   502 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_184b1c05e67e47718e9f58e229c4c748"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:59.824280   500 rpcz_store.cc:269] 1120 03:00:57.613528 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:57.924863 (+311335us) service_pool.cc:224] Handling call
1120 03:00:58.734095 (+809232us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:58.734118 (+    23us) write_op.cc:459] Released schema lock
1120 03:00:59.809223 (+1075105us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:58.713340 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:58.713731 (+   391us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:58.713761 (+    30us) write_op.cc:440] Acquired schema lock
1120 03:00:58.713773 (+    12us) tablet.cc:710] Decoding operations
1120 03:00:58.714040 (+   267us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:58.714104 (+    64us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:58.714124 (+    20us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:00:58.714287 (+   163us) tablet.cc:758] Row locks acquired
1120 03:00:58.714299 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:00:58.714420 (+   121us) write_op.cc:270] Start()
1120 03:00:58.714509 (+    89us) write_op.cc:276] Timestamp: P: 1795143658714385 usec, L: 0
1120 03:00:58.714523 (+    14us) op_driver.cc:350] REPLICATION: starting
1120 03:00:58.714976 (+   453us) log.cc:844] Serialized 527 byte log entry
1120 03:00:58.725284 (+ 10308us) op_driver.cc:466] REPLICATION: finished
1120 03:00:58.725613 (+   329us) write_op.cc:301] APPLY: starting
1120 03:00:58.725669 (+    56us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:58.725758 (+    89us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:58.725854 (+    96us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:58.726006 (+   152us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:58.726156 (+   150us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:00:58.726201 (+    45us) write_op.cc:312] APPLY: finished
1120 03:00:58.726371 (+   170us) log.cc:844] Serialized 23 byte log entry
1120 03:00:58.726518 (+   147us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:58.727114 (+   596us) write_op.cc:459] Released schema lock
1120 03:00:58.727157 (+    43us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"reported_tablets":19,"child_traces":[["op",{"apply.queue_time_us":175,"apply.run_cpu_time_us":1724,"apply.run_wall_time_us":1724,"num_ops":1,"prepare.queue_time_us":99,"prepare.run_cpu_time_us":1900,"prepare.run_wall_time_us":1900,"raft.queue_time_us":195,"raft.run_cpu_time_us":708,"raft.run_wall_time_us":896,"replication_time_us":10662}]]}
I20261120 03:00:59.909050   500 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_58015d78e3d24cd79830293a3ce6176c"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:59.910459   500 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_58015d78e3d24cd79830293a3ce6176c in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:59.914615   495 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_29554130703446b79df44400fb36c94e" } modify_external_catalogs: true
I20261120 03:00:59.915498   499 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_646d629dfeea413a86bc3edf68c7d6ae" } modify_external_catalogs: true
W20261120 03:00:59.918903   496 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.65:40295 (request call id 33) took 2420 ms (client timeout 1000 ms). Trace:
W20261120 03:00:59.919209   496 rpcz_store.cc:269] 1120 03:00:57.498243 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:57.913085 (+414842us) service_pool.cc:224] Handling call
1120 03:00:59.918843 (+2005758us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:59.078521 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:59.078924 (+   403us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:59.078949 (+    25us) write_op.cc:440] Acquired schema lock
1120 03:00:59.078962 (+    13us) tablet.cc:710] Decoding operations
1120 03:00:59.079271 (+   309us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:59.079318 (+    47us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:59.079339 (+    21us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:00:59.079629 (+   290us) tablet.cc:758] Row locks acquired
1120 03:00:59.079644 (+    15us) write_op.cc:260] PREPARE: finished
1120 03:00:59.079795 (+   151us) write_op.cc:270] Start()
1120 03:00:59.079881 (+    86us) write_op.cc:276] Timestamp: P: 1795143659079710 usec, L: 0
1120 03:00:59.079895 (+    14us) op_driver.cc:350] REPLICATION: starting
1120 03:00:59.080564 (+   669us) log.cc:844] Serialized 901 byte log entry
1120 03:00:59.099286 (+ 18722us) op_driver.cc:466] REPLICATION: finished
1120 03:00:59.099597 (+   311us) write_op.cc:301] APPLY: starting
1120 03:00:59.099653 (+    56us) tablet.cc:1451] starting BulkCheckPresence
1120 03:00:59.099761 (+   108us) tablet.cc:1454] finished BulkCheckPresence
1120 03:00:59.099854 (+    93us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:00:59.100121 (+   267us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:00:59.100260 (+   139us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
1120 03:00:59.100301 (+    41us) write_op.cc:312] APPLY: finished
1120 03:00:59.100470 (+   169us) log.cc:844] Serialized 29 byte log entry
1120 03:00:59.100603 (+   133us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:59.100665 (+    62us) write_op.cc:459] Released schema lock
1120 03:00:59.100693 (+    28us) write_op.cc:346] FINISH: Updating metrics
1120 03:00:59.100930 (+   237us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:00:59.100952 (+    22us) write_op.cc:459] Released schema lock
Metrics: {"mutex_wait_us":86704,"reported_tablets":29,"child_traces":[["op",{"apply.queue_time_us":182,"apply.run_cpu_time_us":1296,"apply.run_wall_time_us":1296,"num_ops":2,"prepare.queue_time_us":17695,"prepare.run_cpu_time_us":2400,"prepare.run_wall_time_us":2400,"raft.queue_time_us":178,"raft.run_cpu_time_us":853,"raft.run_wall_time_us":853,"replication_time_us":19303}]]}
W20261120 03:00:59.922701   502 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_184b1c05e67e47718e9f58e229c4c748 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:59.927510   501 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_ee81fc6028814ea3a0cf91abf82d2feb"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:00:59.928922   501 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_ee81fc6028814ea3a0cf91abf82d2feb in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:00:59.930136   750 tablet_service.cc:1542] Processing DeleteTablet for tablet c6a65c1a4a574961bec657ad7c9048e0 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:57 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:59.930869   756 tablet_service.cc:1542] Processing DeleteTablet for tablet 014bd9c83be34c31b90fe4bcf31f1649 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:54 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:00:59.934177  1747 raft_consensus.cc:534] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:59.934610  1747 raft_consensus.cc:556] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:59.885994  1324 raft_consensus.cc:392] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:00:59.936200  1324 raft_consensus.cc:787] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:00:59.937130  1747 leader_election.cc:290] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:00:59.936893  1324 consensus_queue.cc:261] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:00:59.938877  1324 ts_tablet_manager.cc:1480] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.219s	user 0.003s	sys 0.002s
I20261120 03:00:59.949772  1761 ts_tablet_manager.cc:1975] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:00:59.953480  1761 log.cc:1199] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/b549a06d15db449b851fe9549a963416
I20261120 03:00:59.954308  1761 ts_tablet_manager.cc:1996] T b549a06d15db449b851fe9549a963416 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:00:59.965687   774 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b6f0ff2246cd4772af47812b699e93e8" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:00:59.966425   774 raft_consensus.cc:2062] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:00:59.970217   857 leader_election.cc:304] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:00:59.973088  1747 raft_consensus.cc:3024] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader pre-election won for term 2
I20261120 03:00:59.973663  1747 raft_consensus.cc:534] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:00:59.974097  1747 raft_consensus.cc:3296] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Advancing to term 2
I20261120 03:00:59.981030  1597 raft_consensus.cc:534] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:59.981510  1597 raft_consensus.cc:556] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:00:59.983507  1597 leader_election.cc:290] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:00:59.961949  1046 tablet_service.cc:1542] Processing DeleteTablet for tablet a08aa5ddcf1a4061a70275a8eecb1856 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:53 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:59.985949   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ba85dab0d238487181ce5c1592a2c7fd" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:00:59.988962   710 leader_election.cc:343] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:00:59.990842  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ba85dab0d238487181ce5c1592a2c7fd" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:00:59.975263  1760 raft_consensus.cc:534] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:00:59.987885  1055 tablet_service.cc:1542] Processing DeleteTablet for tablet 1885397e42774a89bb347bc776284bc0 with delete_type TABLET_DATA_DELETED (replaced by tablet b549a06d15db449b851fe9549a963416) from {username='slave'} at 127.0.0.1:50744
I20261120 03:00:59.993680  1760 raft_consensus.cc:556] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:00:59.995596  1760 leader_election.cc:290] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:01:00.000615  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 87) took 5658 ms. Trace:
I20261120 03:01:00.000962  1392 rpcz_store.cc:276] 1120 03:00:54.342159 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.343091 (+   932us) service_pool.cc:224] Handling call
1120 03:00:59.722839 (+5379748us) ts_tablet_manager.cc:1197] Deleting tablet ce612f9a121b4b18908724d2726e156f
1120 03:00:59.722883 (+    44us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:59.794600 (+ 71717us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:59.817517 (+ 22917us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.000539 (+183022us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":134904,"tablet-delete.queue_time_us":5379197}
I20261120 03:01:00.003345   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b3f7103b59d443f9bc739083b8e023ce" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:00.010644   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet ce612f9a121b4b18908724d2726e156f (table default.Table_d2bd3060da1545548f422436c9c7c22e [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]) successfully deleted
I20261120 03:01:00.021246  1747 raft_consensus.cc:556] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.024276  1392 tablet_replica.cc:333] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:01:00.025020  1392 raft_consensus.cc:2440] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:00.025441  1392 raft_consensus.cc:2469] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:00.027120  1392 ts_tablet_manager.cc:1962] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:00:59.997856  1073 raft_consensus.cc:3296] T 3d89fb1d7fa14ae8ac252c68338ed4d6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:00.033715  1181 tablet_bootstrap.cc:492] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
W20261120 03:01:00.033631   709 leader_election.cc:343] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:00.034476  1181 ts_tablet_manager.cc:1449] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.318s	user 0.086s	sys 0.069s
W20261120 03:01:00.034168  1012 leader_election.cc:343] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:00.039999  1747 leader_election.cc:290] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 2 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:00.033995   709 leader_election.cc:304] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:00.044925  1761 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:52644 (request call id 125) took 1025 ms. Trace:
I20261120 03:01:00.061911  1076 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b6f0ff2246cd4772af47812b699e93e8" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:00.063570  1076 raft_consensus.cc:2062] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:01:00.064723   498 rpcz_store.cc:275] Call kudu.master.MasterService.ReplaceTablet from 127.0.0.1:45640 (request call id 164) took 2399 ms. Trace:
I20261120 03:01:00.065033   498 rpcz_store.cc:276] 1120 03:00:57.665450 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.899929 (+2234479us) service_pool.cc:224] Handling call
1120 03:01:00.064661 (+164732us) inbound_call.cc:177] Queueing success response
Metrics: {}
I20261120 03:01:00.065788   498 master_service.cc:965] ReplaceTablet: received request to replace tablet a1192a7f050a4f5387a30b2216dec5f0 from {username='slave'} at 127.0.0.1:45640
I20261120 03:01:00.045624  1597 raft_consensus.cc:2966] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20261120 03:01:00.046254   709 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 46675.3us
I20261120 03:01:00.054688   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b6f0ff2246cd4772af47812b699e93e8" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:00.074101  1799 tablet.cc:1872] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20261120 03:01:00.082954  1076 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b6f0ff2246cd4772af47812b699e93e8" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:01:00.083721  1076 raft_consensus.cc:3296] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Advancing to term 2
I20261120 03:01:00.087776  1392 ts_tablet_manager.cc:1975] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:00.088135  1392 log.cc:1199] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/88b9e9692a874ae993dde4b9e2586a70
I20261120 03:01:00.088148  1076 raft_consensus.cc:2062] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234 [term 2 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 2
I20261120 03:01:00.088766  1392 ts_tablet_manager.cc:1996] T 88b9e9692a874ae993dde4b9e2586a70 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:01:00.091202  1800 tablet.cc:1872] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20261120 03:01:00.074544   776 raft_consensus.cc:3296] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Advancing to term 2
I20261120 03:01:00.103273   857 leader_election.cc:304] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:01:00.104137  1791 raft_consensus.cc:3024] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 2 FOLLOWER]: Leader election won for term 2
I20261120 03:01:00.104642  1791 raft_consensus.cc:744] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [term 2 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:01:00.049576  1324 tablet_bootstrap.cc:492] T 5da3d2724dd34aed88e6b08983188004 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:01:00.108126  1791 consensus_queue.cc:238] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.114042  1076 raft_consensus.cc:3296] T dc4133a8f33b4902a6308f9bdd4f91d2 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:00.123494  1180 tablet_bootstrap.cc:700] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440: Time spent opening tablet: real 0.168s	user 0.002s	sys 0.000s
I20261120 03:01:00.124624   776 raft_consensus.cc:2062] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 2 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 2
I20261120 03:01:00.073725  1761 rpcz_store.cc:276] 1120 03:00:59.019409 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.019540 (+   131us) service_pool.cc:224] Handling call
1120 03:00:59.555403 (+535863us) ts_tablet_manager.cc:1197] Deleting tablet b549a06d15db449b851fe9549a963416
1120 03:00:59.555443 (+    40us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:00:59.696090 (+140647us) tablet_metadata.cc:680] Metadata flushed
1120 03:00:59.949639 (+253549us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.044857 (+ 95218us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":535346}
I20261120 03:01:00.094424   774 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b3f7103b59d443f9bc739083b8e023ce" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:00.126977   774 raft_consensus.cc:2062] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:01:00.137008  1011 leader_election.cc:304] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440
I20261120 03:01:00.145694  1180 tablet_bootstrap.cc:654] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:00.151974   495 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_29554130703446b79df44400fb36c94e" } modify_external_catalogs: true
I20261120 03:01:00.159072  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 88) took 5693 ms. Trace:
I20261120 03:01:00.161828  1295 ts_tablet_manager.cc:1975] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:00.161578  1794 raft_consensus.cc:3024] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:00.162396  1794 raft_consensus.cc:534] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:00.162791  1794 raft_consensus.cc:3296] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:00.163864  1794 raft_consensus.cc:556] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.128636   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet b549a06d15db449b851fe9549a963416 (table default.Table_61cb1e03b7e442a8822e32a2295866f3 [id=b5b565c2c2224675baada2ae9b42be54]) successfully deleted
I20261120 03:01:00.164876  1392 rpcz_store.cc:276] 1120 03:00:54.466051 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:54.467762 (+  1711us) service_pool.cc:224] Handling call
1120 03:01:00.002088 (+5534326us) ts_tablet_manager.cc:1197] Deleting tablet 88b9e9692a874ae993dde4b9e2586a70
1120 03:01:00.024146 (+ 22058us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:00.052020 (+ 27874us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.087649 (+ 35629us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.158986 (+ 71337us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":5533754}
I20261120 03:01:00.171805   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 88b9e9692a874ae993dde4b9e2586a70 (table default.Table_e4c5ff8f28ae4efca67257942a4fcb77 [id=b30488f6d14b4c9ab1566f2376be602a]) successfully deleted
I20261120 03:01:00.172894  1392 tablet_replica.cc:333] T 68179488b9bc4e03b65a9f1d434c95b5 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:01:00.177273  1392 raft_consensus.cc:2440] T 68179488b9bc4e03b65a9f1d434c95b5 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:00.177696  1792 raft_consensus.cc:534] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:00.178661  1792 raft_consensus.cc:556] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:00.179270   499 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_646d629dfeea413a86bc3edf68c7d6ae" } modify_external_catalogs: true
I20261120 03:01:00.180413  1076 raft_consensus.cc:1338] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234 [term 2 FOLLOWER]: Refusing update from remote peer 3d30c2428730481db9b0ffd279dd3440: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 2 index: 1. (index mismatch)
I20261120 03:01:00.180609  1792 leader_election.cc:290] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:01:00.181521  1392 raft_consensus.cc:2469] T 68179488b9bc4e03b65a9f1d434c95b5 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:00.183542   777 raft_consensus.cc:1338] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6 [term 2 FOLLOWER]: Refusing update from remote peer 3d30c2428730481db9b0ffd279dd3440: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 2 index: 1. (index mismatch)
I20261120 03:01:00.157039  1761 tablet_replica.cc:333] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:01:00.192834  1761 raft_consensus.cc:2440] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:01:00.193913  1761 raft_consensus.cc:2469] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
W20261120 03:01:00.195252  1011 leader_election.cc:343] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: c6a65c1a4a574961bec657ad7c9048e0
I20261120 03:01:00.201751  1181 raft_consensus.cc:366] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.202440  1181 raft_consensus.cc:392] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:00.202715  1181 raft_consensus.cc:787] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:00.203296  1181 consensus_queue.cc:261] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.205331  1181 ts_tablet_manager.cc:1480] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.168s	user 0.014s	sys 0.000s
I20261120 03:01:00.206228  1181 tablet_bootstrap.cc:492] T f376fdc03b3b4e3680171af6b8574c2e P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:00.177649   496 catalog_manager.cc:5856] T dc4133a8f33b4902a6308f9bdd4f91d2 P 3d30c2428730481db9b0ffd279dd3440 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66). New cstate: current_term: 1 leader_uuid: "3d30c2428730481db9b0ffd279dd3440" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: HEALTHY } } }
W20261120 03:01:00.231230  1585 log.cc:927] Time spent T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b: Append to log took a long time: real 0.061s	user 0.001s	sys 0.000s
W20261120 03:01:00.232244  1151 log.cc:927] Time spent T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Append to log took a long time: real 0.053s	user 0.000s	sys 0.000s
I20261120 03:01:00.232364  1392 ts_tablet_manager.cc:1962] T 68179488b9bc4e03b65a9f1d434c95b5 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
W20261120 03:01:00.234324  1803 log.cc:927] Time spent T dc4133a8f33b4902a6308f9bdd4f91d2 P 823abdbd90ba47bbacad0f8dcff78234: Append to log took a long time: real 0.058s	user 0.000s	sys 0.000s
W20261120 03:01:00.234941  1801 log.cc:927] Time spent T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234: Append to log took a long time: real 0.058s	user 0.001s	sys 0.000s
I20261120 03:01:00.181766   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "c6a65c1a4a574961bec657ad7c9048e0" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:00.189126  1747 consensus_queue.cc:1049] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:00.237810  1747 raft_consensus.cc:534] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:00.238246  1747 raft_consensus.cc:556] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.240082  1747 leader_election.cc:290] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:00.241461  1791 consensus_queue.cc:1049] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20261120 03:01:00.242458  1012 leader_election.cc:343] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:00.242832  1012 leader_election.cc:304] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:01:00.243559  1180 tablet_bootstrap.cc:492] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:01:00.243525  1792 raft_consensus.cc:2966] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:00.243952  1180 ts_tablet_manager.cc:1449] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.544s	user 0.010s	sys 0.000s
I20261120 03:01:00.246214  1761 ts_tablet_manager.cc:1962] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:00.250936   500 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:45640 (request call id 166) took 2110 ms. Trace:
I20261120 03:01:00.250501  1324 tablet_bootstrap.cc:654] T 5da3d2724dd34aed88e6b08983188004 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:00.251251   500 rpcz_store.cc:276] 1120 03:00:58.140755 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.908720 (+1767965us) service_pool.cc:224] Handling call
1120 03:01:00.149202 (+240482us) catalog_manager.cc:2145] Acquired catalog manager lock
1120 03:01:00.150702 (+  1500us) catalog_manager.cc:2191] Created new table and tablet info
1120 03:01:00.250184 (+ 99482us) catalog_manager.cc:2251] Wrote table and tablets to system table
1120 03:01:00.250608 (+   424us) catalog_manager.cc:2285] Inserted table and tablets into CatalogManager maps
1120 03:01:00.250879 (+   271us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:00.152986 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:00.153391 (+   405us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:00.153415 (+    24us) write_op.cc:440] Acquired schema lock
1120 03:01:00.153427 (+    12us) tablet.cc:710] Decoding operations
1120 03:01:00.153693 (+   266us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:00.153733 (+    40us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:00.153765 (+    32us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:01:00.153954 (+   189us) tablet.cc:758] Row locks acquired
1120 03:01:00.153968 (+    14us) write_op.cc:260] PREPARE: finished
1120 03:01:00.154108 (+   140us) write_op.cc:270] Start()
1120 03:01:00.154202 (+    94us) write_op.cc:276] Timestamp: P: 1795143660153994 usec, L: 0
1120 03:01:00.154222 (+    20us) op_driver.cc:350] REPLICATION: starting
1120 03:01:00.154743 (+   521us) log.cc:844] Serialized 433 byte log entry
1120 03:01:00.245313 (+ 90570us) op_driver.cc:466] REPLICATION: finished
1120 03:01:00.245675 (+   362us) write_op.cc:301] APPLY: starting
1120 03:01:00.245730 (+    55us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:00.245839 (+   109us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:00.245938 (+    99us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:00.246386 (+   448us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:00.246524 (+   138us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
1120 03:01:00.246566 (+    42us) write_op.cc:312] APPLY: finished
1120 03:01:00.246746 (+   180us) log.cc:844] Serialized 29 byte log entry
1120 03:01:00.246893 (+   147us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:00.246952 (+    59us) write_op.cc:459] Released schema lock
1120 03:01:00.246986 (+    34us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:00.247808 (+   822us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:00.247833 (+    25us) write_op.cc:459] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":211,"apply.run_cpu_time_us":2083,"apply.run_wall_time_us":2083,"num_ops":2,"prepare.queue_time_us":898,"prepare.run_cpu_time_us":2041,"prepare.run_wall_time_us":2049,"raft.queue_time_us":1210,"raft.run_cpu_time_us":649,"raft.run_wall_time_us":648,"replication_time_us":90993}]]}
I20261120 03:01:00.253361  1794 leader_election.cc:290] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:01:00.253904  1808 raft_consensus.cc:534] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:00.254330  1808 raft_consensus.cc:556] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:00.255836   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b3f7103b59d443f9bc739083b8e023ce" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:01:00.257038  1808 leader_election.cc:290] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
W20261120 03:01:00.257177  1012 leader_election.cc:343] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:00.258937  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9e42a416be884341994d2d5395286798" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:01:00.260833   709 leader_election.cc:343] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:00.264612   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9e42a416be884341994d2d5395286798" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:01:00.266127   710 leader_election.cc:343] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:00.266462   710 leader_election.cc:304] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:00.266567   500 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_ae4914f59f3847c8a2a0ef1d68a7f4b2" } new_table_name: "default.Table_2a81e9814fe34f9bb1902e71fd82fd01" modify_external_catalogs: true
W20261120 03:01:00.231216  1802 log.cc:927] Time spent T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440: Append to log took a long time: real 0.065s	user 0.001s	sys 0.000s
I20261120 03:01:00.270057  1595 raft_consensus.cc:2966] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:00.232584  1295 log.cc:1199] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/4abb67f3819a45b9b05dde04842e2c9d
I20261120 03:01:00.275036  1295 ts_tablet_manager.cc:1996] T 4abb67f3819a45b9b05dde04842e2c9d P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:01:00.284365  1295 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 127) took 1026 ms. Trace:
I20261120 03:01:00.284803  1295 rpcz_store.cc:276] 1120 03:00:59.258149 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.276979 (+ 18830us) service_pool.cc:224] Handling call
1120 03:00:59.665061 (+388082us) ts_tablet_manager.cc:1197] Deleting tablet 4abb67f3819a45b9b05dde04842e2c9d
1120 03:00:59.665097 (+    36us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:00.055081 (+389984us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.161699 (+106618us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.284303 (+122604us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":387446}
I20261120 03:01:00.286170   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 4abb67f3819a45b9b05dde04842e2c9d (table default.Table_69539e86f82e4df2a4b6888aa55293fc [id=0335c1c0df984b0ab347ae9e6d566c5d]) successfully deleted
I20261120 03:01:00.288347   753 tablet_service.cc:1542] Processing DeleteTablet for tablet b549a06d15db449b851fe9549a963416 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:54 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:01:00.289177   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "091dd73d27df4db09ab750747da7584e" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:00.290095   776 raft_consensus.cc:2062] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:01:00.302271   502 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:45640 (request call id 165) took 2185 ms. Trace:
I20261120 03:01:00.302621   502 rpcz_store.cc:276] 1120 03:00:58.116418 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.906771 (+1790353us) service_pool.cc:224] Handling call
1120 03:01:00.174219 (+267448us) catalog_manager.cc:2145] Acquired catalog manager lock
1120 03:01:00.175505 (+  1286us) catalog_manager.cc:2191] Created new table and tablet info
1120 03:01:00.301617 (+126112us) catalog_manager.cc:2251] Wrote table and tablets to system table
1120 03:01:00.301908 (+   291us) catalog_manager.cc:2285] Inserted table and tablets into CatalogManager maps
1120 03:01:00.302211 (+   303us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:00.176778 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:00.177168 (+   390us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:00.177191 (+    23us) write_op.cc:440] Acquired schema lock
1120 03:01:00.177216 (+    25us) tablet.cc:710] Decoding operations
1120 03:01:00.177508 (+   292us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:00.177544 (+    36us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:00.177565 (+    21us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:01:00.177765 (+   200us) tablet.cc:758] Row locks acquired
1120 03:01:00.177779 (+    14us) write_op.cc:260] PREPARE: finished
1120 03:01:00.177889 (+   110us) write_op.cc:270] Start()
1120 03:01:00.177966 (+    77us) write_op.cc:276] Timestamp: P: 1795143660177804 usec, L: 0
1120 03:01:00.177982 (+    16us) op_driver.cc:350] REPLICATION: starting
1120 03:01:00.178467 (+   485us) log.cc:844] Serialized 433 byte log entry
1120 03:01:00.296715 (+118248us) op_driver.cc:466] REPLICATION: finished
1120 03:01:00.298381 (+  1666us) write_op.cc:301] APPLY: starting
1120 03:01:00.298435 (+    54us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:00.298560 (+   125us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:00.298674 (+   114us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:00.299089 (+   415us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:00.299231 (+   142us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
1120 03:01:00.299272 (+    41us) write_op.cc:312] APPLY: finished
1120 03:01:00.299454 (+   182us) log.cc:844] Serialized 29 byte log entry
1120 03:01:00.299608 (+   154us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:00.299687 (+    79us) write_op.cc:459] Released schema lock
1120 03:01:00.299721 (+    34us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:00.299989 (+   268us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:00.300011 (+    22us) write_op.cc:459] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":1515,"apply.run_cpu_time_us":1562,"apply.run_wall_time_us":1561,"num_ops":2,"prepare.queue_time_us":100,"prepare.run_cpu_time_us":1894,"prepare.run_wall_time_us":2322,"replication_time_us":118639}]]}
I20261120 03:01:00.308984   498 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet a1192a7f050a4f5387a30b2216dec5f0
I20261120 03:01:00.310408   498 catalog_manager.cc:6952] ReplaceTablet: tablet a1192a7f050a4f5387a30b2216dec5f0 deleted and replaced by tablet ad4e9e0de4954904842bfd5f15f530f0
I20261120 03:01:00.312561  1595 raft_consensus.cc:534] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 3d30c2428730481db9b0ffd279dd3440)
I20261120 03:01:00.313156  1595 raft_consensus.cc:556] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.314854   898 tablet_service.cc:1542] Processing DeleteTablet for tablet a1192a7f050a4f5387a30b2216dec5f0 with delete_type TABLET_DATA_DELETED (replaced by tablet ad4e9e0de4954904842bfd5f15f530f0) from {username='slave'} at 127.0.0.1:44896
I20261120 03:01:00.315102  1595 leader_election.cc:290] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:01:00.317344   753 tablet_service.cc:1542] Processing DeleteTablet for tablet a1192a7f050a4f5387a30b2216dec5f0 with delete_type TABLET_DATA_DELETED (replaced by tablet ad4e9e0de4954904842bfd5f15f530f0) from {username='slave'} at 127.0.0.1:52644
I20261120 03:01:00.317344  1045 tablet_service.cc:1542] Processing DeleteTablet for tablet a1192a7f050a4f5387a30b2216dec5f0 with delete_type TABLET_DATA_DELETED (replaced by tablet ad4e9e0de4954904842bfd5f15f530f0) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:00.280933  1180 raft_consensus.cc:366] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.318513  1180 raft_consensus.cc:392] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:00.318600   857 leader_election.cc:304] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:01:00.318866  1180 raft_consensus.cc:787] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:00.319506  1791 raft_consensus.cc:3024] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:00.319566  1796 raft_consensus.cc:534] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:00.319885  1791 raft_consensus.cc:534] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:00.320230  1791 raft_consensus.cc:3296] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:00.320043  1796 raft_consensus.cc:556] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.321004  1791 raft_consensus.cc:556] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.322649  1796 leader_election.cc:290] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
W20261120 03:01:00.329418   710 leader_election.cc:343] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Not found: Tablet not found: 88b9e9692a874ae993dde4b9e2586a70
I20261120 03:01:00.330040  1791 leader_election.cc:290] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
W20261120 03:01:00.333010   497 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.65:40295 (request call id 34) took 1706 ms (client timeout 1000 ms). Trace:
W20261120 03:01:00.333480   497 rpcz_store.cc:269] 1120 03:00:58.631934 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.012995 (+381061us) service_pool.cc:224] Handling call
1120 03:01:00.332936 (+1319941us) inbound_call.cc:177] Queueing success response
Metrics: {"reported_tablets":35}
I20261120 03:01:00.319684  1180 consensus_queue.cc:261] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.338205  1180 ts_tablet_manager.cc:1480] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.094s	user 0.006s	sys 0.002s
I20261120 03:01:00.281939  1392 ts_tablet_manager.cc:1975] T 68179488b9bc4e03b65a9f1d434c95b5 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:00.354719  1392 log.cc:1199] T 68179488b9bc4e03b65a9f1d434c95b5 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/68179488b9bc4e03b65a9f1d434c95b5
I20261120 03:01:00.355875  1392 ts_tablet_manager.cc:1996] T 68179488b9bc4e03b65a9f1d434c95b5 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:01:00.358049  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 100) took 3190 ms. Trace:
W20261120 03:01:00.340150   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 88b9e9692a874ae993dde4b9e2586a70 because the tablet was not found. No further retry: Not found: Tablet not found: 88b9e9692a874ae993dde4b9e2586a70
W20261120 03:01:00.358966   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet ce612f9a121b4b18908724d2726e156f because the tablet was not found. No further retry: Not found: Tablet not found: ce612f9a121b4b18908724d2726e156f
I20261120 03:01:00.341001   498 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_523caa128be441e1a52767e3d3318ce0" } new_table_name: "default.Table_3e877edca926443fba86366475176a93" modify_external_catalogs: true
W20261120 03:01:00.367093   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 5c5c0ac869a1419fad9f61ec647027ee because the tablet was not found. No further retry: Not found: Tablet not found: 5c5c0ac869a1419fad9f61ec647027ee
I20261120 03:01:00.368225  1180 tablet_bootstrap.cc:492] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:01:00.369392  1793 raft_consensus.cc:534] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:00.370777   495 catalog_manager.cc:6146] Not sending DeleteTablet requests; no consensus state for tablet 09895b3cffeb443f9ddf693ae09d809c
I20261120 03:01:00.370762  1793 raft_consensus.cc:556] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.371099   495 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:45640 (request call id 167) took 1195 ms. Trace:
W20261120 03:01:00.371446   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 1885397e42774a89bb347bc776284bc0 because the tablet was not found. No further retry: Not found: Tablet not found: 1885397e42774a89bb347bc776284bc0
I20261120 03:01:00.372637  1793 leader_election.cc:290] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:01:00.371373   495 rpcz_store.cc:276] 1120 03:00:59.175333 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.914338 (+739005us) service_pool.cc:224] Handling call
1120 03:01:00.152540 (+238202us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:01:00.187349 (+ 34809us) catalog_manager.cc:2859] Modifying in-memory table state
1120 03:01:00.187483 (+   134us) catalog_manager.cc:2868] Locking tablets
1120 03:01:00.187952 (+   469us) catalog_manager.cc:2882] Removing table and tablets from system table
1120 03:01:00.370449 (+182497us) catalog_manager.cc:2900] Removing table from by-name map
1120 03:01:00.370612 (+   163us) catalog_manager.cc:2916] Committing in-memory state
1120 03:01:00.370698 (+    86us) catalog_manager.cc:2920] Aborting table tasks
1120 03:01:00.371049 (+   351us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:00.189395 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:00.189775 (+   380us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:00.189797 (+    22us) write_op.cc:440] Acquired schema lock
1120 03:01:00.189809 (+    12us) tablet.cc:710] Decoding operations
1120 03:01:00.190150 (+   341us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:00.190184 (+    34us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:00.190214 (+    30us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:01:00.190415 (+   201us) tablet.cc:758] Row locks acquired
1120 03:01:00.190427 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:01:00.190530 (+   103us) write_op.cc:270] Start()
1120 03:01:00.190606 (+    76us) write_op.cc:276] Timestamp: P: 1795143660190448 usec, L: 0
1120 03:01:00.190621 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:01:00.191045 (+   424us) log.cc:844] Serialized 533 byte log entry
1120 03:01:00.339411 (+148366us) op_driver.cc:466] REPLICATION: finished
1120 03:01:00.342098 (+  2687us) write_op.cc:301] APPLY: starting
1120 03:01:00.342155 (+    57us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:00.342274 (+   119us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:00.342374 (+   100us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:00.342613 (+   239us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:00.342740 (+   127us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
1120 03:01:00.342786 (+    46us) write_op.cc:312] APPLY: finished
1120 03:01:00.342970 (+   184us) log.cc:844] Serialized 29 byte log entry
1120 03:01:00.343120 (+   150us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:00.343185 (+    65us) write_op.cc:459] Released schema lock
1120 03:01:00.343228 (+    43us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"child_traces":[["op",{"apply.queue_time_us":2496,"num_ops":2,"prepare.queue_time_us":113,"prepare.run_cpu_time_us":1823,"prepare.run_wall_time_us":1822,"replication_time_us":148691}]]}
W20261120 03:01:00.373567   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 014bd9c83be34c31b90fe4bcf31f1649 because the tablet was not found. No further retry: Not found: Tablet not found: 014bd9c83be34c31b90fe4bcf31f1649
I20261120 03:01:00.374298   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a1192a7f050a4f5387a30b2216dec5f0" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:01:00.375795  1012 leader_election.cc:343] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:01:00.376250   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet a6ae8007455d4be3b50672b9c6b0fd8f because the tablet was not found. No further retry: Not found: Tablet not found: a6ae8007455d4be3b50672b9c6b0fd8f
I20261120 03:01:00.377851  1392 rpcz_store.cc:276] 1120 03:00:57.167192 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:57.167310 (+   118us) service_pool.cc:224] Handling call
1120 03:01:00.172745 (+3005435us) ts_tablet_manager.cc:1197] Deleting tablet 68179488b9bc4e03b65a9f1d434c95b5
1120 03:01:00.172789 (+    44us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:00.257933 (+ 85144us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.281813 (+ 23880us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.357990 (+ 76177us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3004787}
W20261120 03:01:00.380434   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 020be7f7cda0467aab56f657cbe8d0ad because the tablet was not found. No further retry: Not found: Tablet not found: 020be7f7cda0467aab56f657cbe8d0ad
I20261120 03:01:00.383642  1392 tablet_replica.cc:333] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
W20261120 03:01:00.384910   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 68179488b9bc4e03b65a9f1d434c95b5 because the tablet was not found. No further retry: Not found: Tablet not found: 68179488b9bc4e03b65a9f1d434c95b5
I20261120 03:01:00.384963   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 68179488b9bc4e03b65a9f1d434c95b5 (table default.Table_650dc67e0ac74c149a3f64d456767938 [id=cfcd5fea97fb4ab3baff99eb28cd9476]) successfully deleted
I20261120 03:01:00.344712   502 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_a887fc0441b4418eac5a6bca56a83110"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:00.386581  1392 raft_consensus.cc:2440] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Raft consensus shutting down.
W20261120 03:01:00.387269   502 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_a887fc0441b4418eac5a6bca56a83110 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:01:00.388185  1392 raft_consensus.cc:2469] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:00.386101   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b3f7103b59d443f9bc739083b8e023ce" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:00.390208   776 raft_consensus.cc:3296] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:00.391134   776 raft_consensus.cc:2062] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
W20261120 03:01:00.396803   710 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 67411.3us
I20261120 03:01:00.398201   773 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "091dd73d27df4db09ab750747da7584e" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:00.398725   773 raft_consensus.cc:3296] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:00.399605   773 raft_consensus.cc:2062] T 091dd73d27df4db09ab750747da7584e P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:01:00.402717  1392 ts_tablet_manager.cc:1962] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:00.402951   774 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "f376fdc03b3b4e3680171af6b8574c2e" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:00.403395  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "091dd73d27df4db09ab750747da7584e" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:00.403640   774 raft_consensus.cc:2062] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
W20261120 03:01:00.417686   487 catalog_manager.cc:5061] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet a1192a7f050a4f5387a30b2216dec5f0 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet a1192a7f050a4f5387a30b2216dec5f0 already in progress: creating tablet
I20261120 03:01:00.432466  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "f376fdc03b3b4e3680171af6b8574c2e" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:00.434409  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "091dd73d27df4db09ab750747da7584e" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
W20261120 03:01:00.436117   857 leader_election.cc:343] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:00.437173   773 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a1192a7f050a4f5387a30b2216dec5f0" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:00.437856   773 raft_consensus.cc:2062] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:01:00.443337  1180 tablet_bootstrap.cc:654] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:00.460016   501 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:45640 (request call id 169) took 1086 ms. Trace:
I20261120 03:01:00.460567  1761 ts_tablet_manager.cc:1975] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:00.460470   501 rpcz_store.cc:276] 1120 03:00:59.373441 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.927198 (+553757us) service_pool.cc:224] Handling call
1120 03:01:00.173778 (+246580us) catalog_manager.cc:2145] Acquired catalog manager lock
1120 03:01:00.194350 (+ 20572us) catalog_manager.cc:2191] Created new table and tablet info
1120 03:01:00.459390 (+265040us) catalog_manager.cc:2251] Wrote table and tablets to system table
1120 03:01:00.459678 (+   288us) catalog_manager.cc:2285] Inserted table and tablets into CatalogManager maps
1120 03:01:00.459931 (+   253us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:00.195560 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:00.195942 (+   382us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:00.195964 (+    22us) write_op.cc:440] Acquired schema lock
1120 03:01:00.195977 (+    13us) tablet.cc:710] Decoding operations
1120 03:01:00.196242 (+   265us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:00.196276 (+    34us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:00.196301 (+    25us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:01:00.196481 (+   180us) tablet.cc:758] Row locks acquired
1120 03:01:00.196492 (+    11us) write_op.cc:260] PREPARE: finished
1120 03:01:00.196599 (+   107us) write_op.cc:270] Start()
1120 03:01:00.196673 (+    74us) write_op.cc:276] Timestamp: P: 1795143660196512 usec, L: 0
1120 03:01:00.196686 (+    13us) op_driver.cc:350] REPLICATION: starting
1120 03:01:00.197193 (+   507us) log.cc:844] Serialized 433 byte log entry
1120 03:01:00.339652 (+142459us) op_driver.cc:466] REPLICATION: finished
1120 03:01:00.375264 (+ 35612us) write_op.cc:301] APPLY: starting
1120 03:01:00.375324 (+    60us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:00.375423 (+    99us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:00.375526 (+   103us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:00.375918 (+   392us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:00.376047 (+   129us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
1120 03:01:00.376086 (+    39us) write_op.cc:312] APPLY: finished
1120 03:01:00.376316 (+   230us) log.cc:844] Serialized 29 byte log entry
1120 03:01:00.376981 (+   665us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:00.377045 (+    64us) write_op.cc:459] Released schema lock
1120 03:01:00.377075 (+    30us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"child_traces":[["op",{"apply.queue_time_us":35426,"num_ops":2,"prepare.queue_time_us":101,"prepare.run_cpu_time_us":1767,"prepare.run_wall_time_us":1820,"replication_time_us":142893}]]}
I20261120 03:01:00.465786  1180 tablet_bootstrap.cc:492] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:01:00.449846  1392 ts_tablet_manager.cc:1975] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:00.466216  1180 ts_tablet_manager.cc:1449] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.098s	user 0.011s	sys 0.000s
I20261120 03:01:00.466482  1392 log.cc:1199] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/a6ae8007455d4be3b50672b9c6b0fd8f
I20261120 03:01:00.467391  1392 ts_tablet_manager.cc:1996] T a6ae8007455d4be3b50672b9c6b0fd8f P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:01:00.468621  1180 raft_consensus.cc:366] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:00.471971  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 101) took 3302 ms. Trace:
I20261120 03:01:00.482332  1761 log.cc:1199] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/4abb67f3819a45b9b05dde04842e2c9d
I20261120 03:01:00.484108  1392 rpcz_store.cc:276] 1120 03:00:57.170095 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:57.170200 (+   105us) service_pool.cc:224] Handling call
1120 03:01:00.383476 (+3213276us) ts_tablet_manager.cc:1197] Deleting tablet a6ae8007455d4be3b50672b9c6b0fd8f
1120 03:01:00.383521 (+    45us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:00.425171 (+ 41650us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.449725 (+ 24554us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.471901 (+ 22176us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":1200,"tablet-delete.queue_time_us":3212706}
I20261120 03:01:00.485333   494 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet d736d65c3e0b419084b4934dad83f16a (table default.Table_04ddab7f706b43e7ba12fa247b3ca933 [id=9f571a69327a498e89f323c9ac0eee34]). Expected version 1 got 0
I20261120 03:01:00.488572  1761 ts_tablet_manager.cc:1996] T 4abb67f3819a45b9b05dde04842e2c9d P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:01:00.486904  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 102) took 3315 ms. Trace:
I20261120 03:01:00.491608  1392 rpcz_store.cc:276] 1120 03:00:57.171095 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:57.171193 (+    98us) service_pool.cc:224] Handling call
1120 03:01:00.486493 (+3315300us) ts_tablet_manager.cc:1197] Deleting tablet 014bd9c83be34c31b90fe4bcf31f1649
1120 03:01:00.486543 (+    50us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:00.486838 (+   295us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3314687}
I20261120 03:01:00.469233  1180 raft_consensus.cc:392] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
W20261120 03:01:00.495452   857 leader_election.cc:343] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:00.495626  1180 raft_consensus.cc:787] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:00.496129  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 103) took 3320 ms. Trace:
W20261120 03:01:00.496757   857 leader_election.cc:343] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:00.496495  1392 rpcz_store.cc:276] 1120 03:00:57.175941 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:57.176038 (+    97us) service_pool.cc:224] Handling call
1120 03:01:00.495878 (+3319840us) ts_tablet_manager.cc:1197] Deleting tablet fa57d0c282964e37b9133ea57f458a51
1120 03:01:00.495922 (+    44us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:00.496074 (+   152us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3319209}
I20261120 03:01:00.498224  1392 tablet_replica.cc:333] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:01:00.499066  1392 raft_consensus.cc:2440] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:00.496434  1180 consensus_queue.cc:261] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:00.500494  1011 leader_election.cc:304] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440
I20261120 03:01:00.500861  1180 ts_tablet_manager.cc:1480] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.034s	user 0.004s	sys 0.000s
I20261120 03:01:00.501320  1793 raft_consensus.cc:3024] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:00.501708  1180 tablet_bootstrap.cc:492] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:01:00.501645  1793 raft_consensus.cc:534] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:00.500532  1392 raft_consensus.cc:2469] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:00.501983  1793 raft_consensus.cc:3296] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:00.502765  1793 raft_consensus.cc:556] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.503623  1392 ts_tablet_manager.cc:1962] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:00.514961  1793 leader_election.cc:290] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:01:00.515170   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a1192a7f050a4f5387a30b2216dec5f0" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
W20261120 03:01:00.516619  1012 leader_election.cc:343] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:00.524789  1181 tablet_bootstrap.cc:654] T f376fdc03b3b4e3680171af6b8574c2e P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:00.527845  1180 tablet_bootstrap.cc:654] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:00.530753  1821 tablet.cc:1872] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20261120 03:01:00.485867   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet a6ae8007455d4be3b50672b9c6b0fd8f (table default.Table_f73e1375172b4e9999e9906c1491c0d5 [id=96692554644a476d87e6bdcaa3bd847c]) successfully deleted
W20261120 03:01:00.557145   486 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 71310.8us
W20261120 03:01:00.562769   486 catalog_manager.cc:5061] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 014bd9c83be34c31b90fe4bcf31f1649 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 014bd9c83be34c31b90fe4bcf31f1649 already in progress: creating tablet
I20261120 03:01:00.538938   501 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_902789719933401f96796eebfca3fc97"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:01:00.565907   501 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_902789719933401f96796eebfca3fc97 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:01:00.567854  1180 tablet_bootstrap.cc:492] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:01:00.542728   495 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_664fa9b65c084e0ba9c360a594a48b66"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:00.568259  1180 ts_tablet_manager.cc:1449] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.067s	user 0.007s	sys 0.003s
W20261120 03:01:00.569191   495 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_664fa9b65c084e0ba9c360a594a48b66 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:01:00.570640  1180 raft_consensus.cc:366] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.571249  1180 raft_consensus.cc:392] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:00.571506  1180 raft_consensus.cc:787] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:00.572211  1180 consensus_queue.cc:261] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.576601   497 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_69539e86f82e4df2a4b6888aa55293fc" } modify_external_catalogs: true
I20261120 03:01:00.577101   497 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_69539e86f82e4df2a4b6888aa55293fc" } modify_external_catalogs: true
I20261120 03:01:00.585418  1180 ts_tablet_manager.cc:1480] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.017s	user 0.005s	sys 0.001s
I20261120 03:01:00.586440  1180 tablet_bootstrap.cc:492] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:01:00.598985   494 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 0db7db25d779459ebde532f938493975 (table default.Table_9fe8b8c914534e4ca5602de7c1b1d280 [id=f3b5f7a4e5494e82a472d0025df96e0c]). Expected version 2 got 0
I20261120 03:01:00.599884   494 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
I20261120 03:01:00.602135  1180 tablet_bootstrap.cc:654] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:00.628409  1180 tablet_bootstrap.cc:492] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:01:00.628844  1180 ts_tablet_manager.cc:1449] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.043s	user 0.009s	sys 0.000s
W20261120 03:01:00.628779  1011 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 128359us
I20261120 03:01:00.631215  1180 raft_consensus.cc:366] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.631831  1180 raft_consensus.cc:392] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:00.632091  1180 raft_consensus.cc:787] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:00.633071  1180 consensus_queue.cc:261] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.635070  1180 ts_tablet_manager.cc:1480] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.006s	user 0.007s	sys 0.000s
I20261120 03:01:00.636240  1392 ts_tablet_manager.cc:1975] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:00.636698  1392 log.cc:1199] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/c6a65c1a4a574961bec657ad7c9048e0
I20261120 03:01:00.637267  1392 ts_tablet_manager.cc:1996] T c6a65c1a4a574961bec657ad7c9048e0 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:01:00.644834  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 118) took 1674 ms. Trace:
I20261120 03:01:00.648278  1180 tablet_bootstrap.cc:492] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:01:00.653743  1180 tablet_bootstrap.cc:654] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
W20261120 03:01:00.659168   488 catalog_manager.cc:4863] Async tablet task AlterTable RPC for tablet 8935b23c661c4c1ca7bde7c893927502 (table default.Table_92f6cc71382f4deba2472c49ae8f6a36 [id=51df6e371d3441578e57ea5597ee81a6], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 8935b23c661c4c1ca7bde7c893927502 consensus state has no leader
I20261120 03:01:00.645251  1392 rpcz_store.cc:276] 1120 03:00:58.970694 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:58.970837 (+   143us) service_pool.cc:224] Handling call
1120 03:01:00.498021 (+1527184us) ts_tablet_manager.cc:1197] Deleting tablet c6a65c1a4a574961bec657ad7c9048e0
1120 03:01:00.498063 (+    42us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:00.629751 (+131688us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.636124 (+  6373us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.644766 (+  8642us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1526599}
I20261120 03:01:00.666847   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet c6a65c1a4a574961bec657ad7c9048e0 (table default.Table_dd2f7617ff0e401b9b862d5991110dd5 [id=3955f41b6d4d41bf8ba2c81e4ac7f4a8]) successfully deleted
I20261120 03:01:00.667212  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 120) took 1603 ms. Trace:
I20261120 03:01:00.667574  1392 rpcz_store.cc:276] 1120 03:00:59.063818 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.063964 (+   146us) service_pool.cc:224] Handling call
1120 03:01:00.666793 (+1602829us) ts_tablet_manager.cc:1197] Deleting tablet d753dd170cab4f18980e85707c4093e9
1120 03:01:00.666840 (+    47us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:00.667151 (+   311us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1602238}
W20261120 03:01:00.670764   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet d753dd170cab4f18980e85707c4093e9 because the tablet was not found. No further retry: Not found: Tablet not found: d753dd170cab4f18980e85707c4093e9
I20261120 03:01:00.671157  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 121) took 1605 ms. Trace:
I20261120 03:01:00.671520  1392 rpcz_store.cc:276] 1120 03:00:59.065297 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.068754 (+  3457us) service_pool.cc:224] Handling call
1120 03:01:00.670903 (+1602149us) ts_tablet_manager.cc:1197] Deleting tablet 9a31431495274c648625b75900b4ac47
1120 03:01:00.670948 (+    45us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:00.671105 (+   157us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1601652}
I20261120 03:01:00.674737  1180 tablet_bootstrap.cc:492] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:01:00.675110  1180 ts_tablet_manager.cc:1449] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.027s	user 0.009s	sys 0.003s
W20261120 03:01:00.675469   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 9a31431495274c648625b75900b4ac47 because the tablet was not found. No further retry: Not found: Tablet not found: 9a31431495274c648625b75900b4ac47
I20261120 03:01:00.675830  1392 tablet_replica.cc:333] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:01:00.677327  1180 raft_consensus.cc:366] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.677932  1180 raft_consensus.cc:392] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:00.678206  1180 raft_consensus.cc:787] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:00.678618  1392 raft_consensus.cc:2440] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:00.679157  1392 raft_consensus.cc:2469] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:00.680969  1392 ts_tablet_manager.cc:1962] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:00.684562  1180 consensus_queue.cc:261] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.686527  1180 ts_tablet_manager.cc:1480] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.011s	user 0.006s	sys 0.000s
I20261120 03:01:00.687340  1180 tablet_bootstrap.cc:492] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:01:00.694890  1791 raft_consensus.cc:534] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:00.695361  1791 raft_consensus.cc:556] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.697270  1791 leader_election.cc:290] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
W20261120 03:01:00.706467  1825 log.cc:927] Time spent T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f: Append to log took a long time: real 0.061s	user 0.011s	sys 0.005s
I20261120 03:01:00.711242  1180 tablet_bootstrap.cc:654] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:00.715011  1791 raft_consensus.cc:534] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:00.715588  1791 raft_consensus.cc:556] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
W20261120 03:01:00.717980   709 leader_election.cc:343] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 2 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Not found: Tablet not found: 88b9e9692a874ae993dde4b9e2586a70
I20261120 03:01:00.722323   709 leader_election.cc:304] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:00.725533  1791 leader_election.cc:290] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:00.737216  1180 tablet_bootstrap.cc:492] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:01:00.737738  1180 ts_tablet_manager.cc:1449] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.051s	user 0.013s	sys 0.000s
I20261120 03:01:00.740418  1180 raft_consensus.cc:366] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:00.741186  1180 raft_consensus.cc:392] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:00.741484  1180 raft_consensus.cc:787] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:00.742285  1180 consensus_queue.cc:261] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:00.746384   857 leader_election.cc:304] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:00.747336  1791 raft_consensus.cc:3024] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:00.747665  1791 raft_consensus.cc:534] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:00.747973  1791 raft_consensus.cc:3296] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:00.748723  1791 raft_consensus.cc:556] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.751663   857 leader_election.cc:304] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:00.752398  1796 raft_consensus.cc:3024] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:00.760425  1791 leader_election.cc:290] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:00.767210  1761 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:52644 (request call id 126) took 1516 ms. Trace:
I20261120 03:01:00.767632  1761 rpcz_store.cc:276] 1120 03:00:59.251072 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.251203 (+   131us) service_pool.cc:224] Handling call
1120 03:01:00.129528 (+878325us) ts_tablet_manager.cc:1197] Deleting tablet 4abb67f3819a45b9b05dde04842e2c9d
1120 03:01:00.152070 (+ 22542us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:00.397594 (+245524us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.460421 (+ 62827us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.766998 (+306577us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":736,"tablet-delete.queue_time_us":877758}
I20261120 03:01:00.772001  1180 ts_tablet_manager.cc:1480] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.034s	user 0.005s	sys 0.000s
W20261120 03:01:00.772818  1324 thread.cc:642] log-alloc [worker] (thread pool) Time spent creating pthread: real 0.519s	user 0.009s	sys 0.009s
W20261120 03:01:00.774360  1324 thread.cc:609] log-alloc [worker] (thread pool) Time spent starting thread: real 0.521s	user 0.010s	sys 0.009s
I20261120 03:01:00.774915  1180 tablet_bootstrap.cc:492] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
W20261120 03:01:00.779944   709 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 62304.8us
I20261120 03:01:00.780659  1808 raft_consensus.cc:2966] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20261120 03:01:00.789206  1761 tablet_replica.cc:333] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:01:00.794795  1180 tablet_bootstrap.cc:654] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:00.798341  1761 raft_consensus.cc:2440] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:01:00.802366  1761 raft_consensus.cc:2469] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:00.809104  1796 raft_consensus.cc:744] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:01:00.809962  1796 consensus_queue.cc:238] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.769850  1392 ts_tablet_manager.cc:1975] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:00.816256  1392 log.cc:1199] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/020be7f7cda0467aab56f657cbe8d0ad
I20261120 03:01:00.818485  1392 ts_tablet_manager.cc:1996] T 020be7f7cda0467aab56f657cbe8d0ad P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:01:00.820417  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 124) took 1748 ms. Trace:
I20261120 03:01:00.820711  1392 rpcz_store.cc:276] 1120 03:00:59.071838 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.071981 (+   143us) service_pool.cc:224] Handling call
1120 03:01:00.675659 (+1603678us) ts_tablet_manager.cc:1197] Deleting tablet 020be7f7cda0467aab56f657cbe8d0ad
1120 03:01:00.675705 (+    46us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:00.744281 (+ 68576us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.769705 (+ 25424us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.820359 (+ 50654us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1603195}
I20261120 03:01:00.821925   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 020be7f7cda0467aab56f657cbe8d0ad (table default.Table_d1737969bdd644a6ab39da121eeedd60 [id=ed325a58ff124ba9a6b1d05bc901793b]) successfully deleted
I20261120 03:01:00.820376   774 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b3f7103b59d443f9bc739083b8e023ce" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:00.827467   774 raft_consensus.cc:2628] T b3f7103b59d443f9bc739083b8e023ce P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 823abdbd90ba47bbacad0f8dcff78234 in this term.
I20261120 03:01:00.800493  1324 tablet_bootstrap.cc:492] T 5da3d2724dd34aed88e6b08983188004 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:01:00.830539  1324 ts_tablet_manager.cc:1449] T 5da3d2724dd34aed88e6b08983188004 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.781s	user 0.017s	sys 0.014s
I20261120 03:01:00.831923  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 119) took 1773 ms. Trace:
I20261120 03:01:00.832226  1392 rpcz_store.cc:276] 1120 03:00:59.059035 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.059305 (+   270us) service_pool.cc:224] Handling call
1120 03:01:00.831655 (+1772350us) ts_tablet_manager.cc:1197] Deleting tablet a642c32c1b6043978516d97efa2347a3
1120 03:01:00.831697 (+    42us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:00.831868 (+   171us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1756431}
W20261120 03:01:00.833400   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet a642c32c1b6043978516d97efa2347a3 because the tablet was not found. No further retry: Not found: Tablet not found: a642c32c1b6043978516d97efa2347a3
I20261120 03:01:00.833819  1324 raft_consensus.cc:366] T 5da3d2724dd34aed88e6b08983188004 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:00.834524  1324 raft_consensus.cc:392] T 5da3d2724dd34aed88e6b08983188004 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:00.834785  1324 raft_consensus.cc:787] T 5da3d2724dd34aed88e6b08983188004 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:01:00.835323  1324 consensus_queue.cc:261] T 5da3d2724dd34aed88e6b08983188004 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:00.836619  1782 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 18331bef5c884dcab9dbd26bc2c82273. Latest consensus state: current_term: 3 leader_uuid: "18331bef5c884dcab9dbd26bc2c82273" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:01:00.837308  1782 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:01:00.837812  1817 raft_consensus.cc:534] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 18331bef5c884dcab9dbd26bc2c82273)
I20261120 03:01:00.838366  1817 raft_consensus.cc:556] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:01:00.840744  1392 tablet_replica.cc:333] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:01:00.843230  1180 tablet_bootstrap.cc:492] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:01:00.843607  1180 ts_tablet_manager.cc:1449] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.069s	user 0.014s	sys 0.010s
I20261120 03:01:00.841455  1392 raft_consensus.cc:2440] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:00.846020  1180 raft_consensus.cc:366] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.846552  1392 raft_consensus.cc:2469] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:00.846657  1180 raft_consensus.cc:392] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:00.848366  1392 ts_tablet_manager.cc:1962] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:00.685163  1181 tablet_bootstrap.cc:492] T f376fdc03b3b4e3680171af6b8574c2e P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:00.849992  1181 ts_tablet_manager.cc:1449] T f376fdc03b3b4e3680171af6b8574c2e P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.644s	user 0.010s	sys 0.000s
I20261120 03:01:00.850162  1817 leader_election.cc:290] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers e102e07459954b4ea13b65ab9fd5d18b (127.0.96.126:44467), 18331bef5c884dcab9dbd26bc2c82273 (127.0.96.125:37411)
I20261120 03:01:00.852437  1181 raft_consensus.cc:366] T f376fdc03b3b4e3680171af6b8574c2e P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.853058  1181 raft_consensus.cc:392] T f376fdc03b3b4e3680171af6b8574c2e P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:00.853307  1181 raft_consensus.cc:787] T f376fdc03b3b4e3680171af6b8574c2e P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:00.856241   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "f376fdc03b3b4e3680171af6b8574c2e" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:00.846887  1180 raft_consensus.cc:787] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:00.856844   777 raft_consensus.cc:3296] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:00.857185  1180 consensus_queue.cc:261] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:00.859143  1180 ts_tablet_manager.cc:1480] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.015s	user 0.005s	sys 0.000s
I20261120 03:01:00.859961  1180 tablet_bootstrap.cc:492] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
W20261120 03:01:00.865532   857 leader_election.cc:343] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: c6a65c1a4a574961bec657ad7c9048e0
I20261120 03:01:00.868423  1837 consensus_queue.cc:1049] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:00.868774   970 heartbeater.cc:345] Connected to a master server at 127.0.96.124:38133
I20261120 03:01:00.871003  1699 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:33064 (request call id 0) took 1894 ms. Trace:
I20261120 03:01:00.875025  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b3f7103b59d443f9bc739083b8e023ce" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:00.875780  1073 raft_consensus.cc:2628] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 823abdbd90ba47bbacad0f8dcff78234 in this term.
I20261120 03:01:00.877768  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "f376fdc03b3b4e3680171af6b8574c2e" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:01:00.879441  1392 ts_tablet_manager.cc:1975] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:00.879797  1392 log.cc:1199] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/200b234da897419a82bf5844620adf24
I20261120 03:01:00.880367  1392 ts_tablet_manager.cc:1996] T 200b234da897419a82bf5844620adf24 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:01:00.894889  1180 tablet_bootstrap.cc:654] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:00.902633  1847 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
I20261120 03:01:00.904558  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 123) took 1835 ms. Trace:
I20261120 03:01:00.907073  1392 rpcz_store.cc:276] 1120 03:00:59.069568 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.072933 (+  3365us) service_pool.cc:224] Handling call
1120 03:01:00.840596 (+1767663us) ts_tablet_manager.cc:1197] Deleting tablet 200b234da897419a82bf5844620adf24
1120 03:01:00.840637 (+    41us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:00.873183 (+ 32546us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.879333 (+  6150us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.904497 (+ 25164us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":5640,"tablet-delete.queue_time_us":1767195}
I20261120 03:01:00.908291  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 122) took 1840 ms. Trace:
I20261120 03:01:00.908576  1392 rpcz_store.cc:276] 1120 03:00:59.068227 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.068434 (+   207us) service_pool.cc:224] Handling call
1120 03:01:00.908034 (+1839600us) ts_tablet_manager.cc:1197] Deleting tablet c3b56d0ccab140c28a37808d9d5902c6
1120 03:01:00.908077 (+    43us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:00.908241 (+   164us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1793589}
I20261120 03:01:00.909579  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 125) took 1831 ms. Trace:
I20261120 03:01:00.909842  1392 rpcz_store.cc:276] 1120 03:00:59.078608 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.079045 (+   437us) service_pool.cc:224] Handling call
1120 03:01:00.909292 (+1830247us) ts_tablet_manager.cc:1197] Deleting tablet b549a06d15db449b851fe9549a963416
1120 03:01:00.909329 (+    37us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:00.909536 (+   207us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":634,"tablet-delete.queue_time_us":1786382}
I20261120 03:01:00.910805  1392 tablet_replica.cc:333] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:01:00.911015   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 200b234da897419a82bf5844620adf24 (table default.Table_5303db2882934be1bfc2251ce51f2c8d [id=ae52d3d892e24bc38a8ff7d6db755c2d]) successfully deleted
I20261120 03:01:00.911262   512 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" candidate_term: 4 candidate_status { last_received { term: 3 index: 190 } } ignore_live_leader: false dest_uuid: "18331bef5c884dcab9dbd26bc2c82273" is_pre_election: true
W20261120 03:01:00.911643   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet c3b56d0ccab140c28a37808d9d5902c6 because the tablet was not found. No further retry: Not found: Tablet not found: c3b56d0ccab140c28a37808d9d5902c6
W20261120 03:01:00.912180   486 catalog_manager.cc:5061] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet b549a06d15db449b851fe9549a963416 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet b549a06d15db449b851fe9549a963416 already in progress: creating tablet
I20261120 03:01:00.914795  1392 raft_consensus.cc:2440] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:00.915275  1392 raft_consensus.cc:2469] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:00.915238   436 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" candidate_term: 4 candidate_status { last_received { term: 3 index: 190 } } ignore_live_leader: false dest_uuid: "e102e07459954b4ea13b65ab9fd5d18b" is_pre_election: true
I20261120 03:01:00.917071  1392 ts_tablet_manager.cc:1962] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:00.917102  1641 leader_election.cc:304] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2599fd9eef7d4ce990c40e2356a37c3f; no voters: 18331bef5c884dcab9dbd26bc2c82273, e102e07459954b4ea13b65ab9fd5d18b
I20261120 03:01:00.918465  1817 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20261120 03:01:00.918917   495 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_43f3d907270e4093a84918b0a505ecc9" } new_table_name: "default.Table_a96b0ac352c545da9b955502704e266b" modify_external_catalogs: true
I20261120 03:01:00.927260   502 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_5415c198c7814e0f988683d59c9181d3"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:00.929039  1180 tablet_bootstrap.cc:492] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
W20261120 03:01:00.929009   502 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_5415c198c7814e0f988683d59c9181d3 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:01:00.929435  1180 ts_tablet_manager.cc:1449] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.070s	user 0.009s	sys 0.000s
I20261120 03:01:00.930370  1392 ts_tablet_manager.cc:1975] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:00.930685  1392 log.cc:1199] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/4abb67f3819a45b9b05dde04842e2c9d
I20261120 03:01:00.931326  1392 ts_tablet_manager.cc:1996] T 4abb67f3819a45b9b05dde04842e2c9d P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:01:00.931805  1180 raft_consensus.cc:366] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:00.932399  1180 raft_consensus.cc:392] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:00.932650  1180 raft_consensus.cc:787] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:00.933148  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 126) took 1683 ms. Trace:
I20261120 03:01:00.933424  1392 rpcz_store.cc:276] 1120 03:00:59.249790 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.249901 (+   111us) service_pool.cc:224] Handling call
1120 03:01:00.910644 (+1660743us) ts_tablet_manager.cc:1197] Deleting tablet 4abb67f3819a45b9b05dde04842e2c9d
1120 03:01:00.910684 (+    40us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:00.924018 (+ 13334us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.930262 (+  6244us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:00.933095 (+  2833us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1626420}
I20261120 03:01:00.934659   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 4abb67f3819a45b9b05dde04842e2c9d (table default.Table_69539e86f82e4df2a4b6888aa55293fc [id=0335c1c0df984b0ab347ae9e6d566c5d]) successfully deleted
I20261120 03:01:00.935051   501 master_service.cc:965] ReplaceTablet: received request to replace tablet 9e42a416be884341994d2d5395286798 from {username='slave'} at 127.0.0.1:45640
I20261120 03:01:00.935690  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 134) took 1099 ms. Trace:
I20261120 03:01:00.935981  1392 rpcz_store.cc:276] 1120 03:00:59.836707 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.961168 (+124461us) service_pool.cc:224] Handling call
1120 03:01:00.935446 (+974278us) ts_tablet_manager.cc:1197] Deleting tablet a08aa5ddcf1a4061a70275a8eecb1856
1120 03:01:00.935484 (+    38us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:00.935644 (+   160us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":944699}
I20261120 03:01:00.936487  1180 consensus_queue.cc:261] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
W20261120 03:01:00.937168   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet a08aa5ddcf1a4061a70275a8eecb1856 because the tablet was not found. No further retry: Not found: Tablet not found: a08aa5ddcf1a4061a70275a8eecb1856
I20261120 03:01:00.938453  1180 ts_tablet_manager.cc:1480] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.009s	user 0.007s	sys 0.000s
I20261120 03:01:00.939296  1180 tablet_bootstrap.cc:492] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:01:00.830004  1761 ts_tablet_manager.cc:1962] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:00.944556  1180 tablet_bootstrap.cc:654] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:00.948815   857 leader_election.cc:304] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:00.950054  1837 raft_consensus.cc:3296] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:00.950721  1392 tablet_replica.cc:333] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:01:00.950767  1837 raft_consensus.cc:2966] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:00.954931  1392 raft_consensus.cc:2440] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234 [term 2 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:00.955428  1392 raft_consensus.cc:2469] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234 [term 2 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:00.957149  1392 ts_tablet_manager.cc:1962] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:00.959935  1679 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" instance_seqno: 1795143644988797) as {username='slave'} at 127.0.96.66:42371; Asking this server to re-register.
I20261120 03:01:00.963225   970 heartbeater.cc:462] Registering TS with master...
I20261120 03:01:00.968336  1180 tablet_bootstrap.cc:492] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
W20261120 03:01:00.971804   857 leader_election.cc:343] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Not found: Tablet not found: c6a65c1a4a574961bec657ad7c9048e0
I20261120 03:01:00.978343   857 leader_election.cc:304] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
W20261120 03:01:00.979700   857 consensus_peers.cc:597] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:01:00.980387  1180 ts_tablet_manager.cc:1449] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.041s	user 0.015s	sys 0.000s
I20261120 03:01:00.980830  1796 raft_consensus.cc:2966] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:00.989609  1180 raft_consensus.cc:366] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:00.990264  1180 raft_consensus.cc:392] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:00.990541  1180 raft_consensus.cc:787] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:00.991076  1180 consensus_queue.cc:261] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:00.992964  1180 ts_tablet_manager.cc:1480] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.012s	user 0.003s	sys 0.000s
I20261120 03:01:01.000560  1679 ts_manager.cc:210] Registered new tserver with Master: 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:01:01.006919  1181 consensus_queue.cc:261] T f376fdc03b3b4e3680171af6b8574c2e P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:01.007823  1073 raft_consensus.cc:3296] T f376fdc03b3b4e3680171af6b8574c2e P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:01.009755  1073 raft_consensus.cc:2062] T f376fdc03b3b4e3680171af6b8574c2e P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:01:01.020202  1011 leader_election.cc:304] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440
I20261120 03:01:01.021164  1794 raft_consensus.cc:3024] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:01.026458  1796 raft_consensus.cc:534] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:01.026923  1796 raft_consensus.cc:556] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:01.029249   777 raft_consensus.cc:2062] T f376fdc03b3b4e3680171af6b8574c2e P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:01:01.034983  1392 ts_tablet_manager.cc:1975] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 2.2
I20261120 03:01:01.035334  1392 log.cc:1199] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/1885397e42774a89bb347bc776284bc0
I20261120 03:01:01.035991  1392 ts_tablet_manager.cc:1996] T 1885397e42774a89bb347bc776284bc0 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:01:01.040668  1392 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:50744 (request call id 133) took 1216 ms. Trace:
I20261120 03:01:01.040966  1392 rpcz_store.cc:276] 1120 03:00:59.824069 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.961367 (+137298us) service_pool.cc:224] Handling call
1120 03:01:00.950575 (+989208us) ts_tablet_manager.cc:1197] Deleting tablet 1885397e42774a89bb347bc776284bc0
1120 03:01:00.950615 (+    40us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:01.011391 (+ 60776us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:01.034863 (+ 23472us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:01.040614 (+  5751us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":956286}
I20261120 03:01:01.042567   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 4abb67f3819a45b9b05dde04842e2c9d (table default.Table_69539e86f82e4df2a4b6888aa55293fc [id=0335c1c0df984b0ab347ae9e6d566c5d]) successfully deleted
I20261120 03:01:01.062227   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 1885397e42774a89bb347bc776284bc0 (table default.Table_61cb1e03b7e442a8822e32a2295866f3 [id=b5b565c2c2224675baada2ae9b42be54]) successfully deleted
I20261120 03:01:01.063081  1392 tablet_replica.cc:333] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:01:01.063795  1392 raft_consensus.cc:2440] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:01.064244  1392 raft_consensus.cc:2469] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:01.065924  1392 ts_tablet_manager.cc:1962] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:01.076438  1794 raft_consensus.cc:744] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Becoming Leader. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Running, Role: LEADER
I20261120 03:01:01.077348  1794 consensus_queue.cc:238] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:01.079581   773 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a1192a7f050a4f5387a30b2216dec5f0" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:01.080209   773 raft_consensus.cc:3296] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:01.081161   773 raft_consensus.cc:2062] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
I20261120 03:01:01.073889  1796 leader_election.cc:290] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:01.094486  1181 ts_tablet_manager.cc:1480] T f376fdc03b3b4e3680171af6b8574c2e P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.244s	user 0.006s	sys 0.000s
I20261120 03:01:01.096959  1850 tablet.cc:1867] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440: Skipping requested alter to schema version 2, tablet already version 2
I20261120 03:01:01.098641  1850 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.AlterSchema from 127.0.0.1:44896 (request call id 144) took 1200 ms. Trace:
W20261120 03:01:01.081911   710 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 110561us
I20261120 03:01:01.131704  1324 ts_tablet_manager.cc:1480] T 5da3d2724dd34aed88e6b08983188004 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.301s	user 0.005s	sys 0.001s
I20261120 03:01:01.132831  1324 tablet_bootstrap.cc:492] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:01:01.132678   502 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_5cbde2434cf04febac447e8bb2c628d5"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:01:01.138528   502 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_5cbde2434cf04febac447e8bb2c628d5 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:01:01.155308   857 leader_election.cc:304] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:01:01.156414  1853 raft_consensus.cc:3024] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:01.156848  1853 raft_consensus.cc:744] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:01:01.157625  1853 consensus_queue.cc:238] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:01.164933   773 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1faba22e8e38418383b86e42e904a761" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:01.165629   773 raft_consensus.cc:2062] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:01:01.167985  1180 tablet_bootstrap.cc:492] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
W20261120 03:01:01.170281   709 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 81532.1us
W20261120 03:01:01.100273  1011 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 80162.3us
I20261120 03:01:01.133184  1392 ts_tablet_manager.cc:1975] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:01.172084  1392 log.cc:1199] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/a1192a7f050a4f5387a30b2216dec5f0
I20261120 03:01:01.172732  1392 ts_tablet_manager.cc:1996] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:01:01.176085  1679 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" instance_seqno: 1795143646840676) as {username='slave'} at 127.0.96.67:41041; Asking this server to re-register.
I20261120 03:01:01.178632   496 catalog_manager.cc:5856] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440 reported cstate change: term changed from 0 to 2, leader changed from <none> to 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66). New cstate: current_term: 2 leader_uuid: "3d30c2428730481db9b0ffd279dd3440" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: HEALTHY } } }
I20261120 03:01:01.179818   496 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
I20261120 03:01:01.183375   898 tablet_service.cc:1495] Processing CreateTablet for tablet 5ea1646a2ab84345a77fd7ffd249003a (DEFAULT_TABLE table=default.Table_150226bfe577411d88c38a63b1af73e5 [id=34604d7af7314b66b2b9605b9bdeb724]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:01.184394   898 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5ea1646a2ab84345a77fd7ffd249003a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:01.194159  1045 tablet_service.cc:1495] Processing CreateTablet for tablet 5ea1646a2ab84345a77fd7ffd249003a (DEFAULT_TABLE table=default.Table_150226bfe577411d88c38a63b1af73e5 [id=34604d7af7314b66b2b9605b9bdeb724]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:01.195225  1045 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5ea1646a2ab84345a77fd7ffd249003a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:01.202683   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet a1192a7f050a4f5387a30b2216dec5f0 (table default.Table_646d629dfeea413a86bc3edf68c7d6ae [id=aa0b779e8ea84d928af40806a7913f1a]) successfully deleted
I20261120 03:01:01.204412   753 tablet_service.cc:1495] Processing CreateTablet for tablet 5ea1646a2ab84345a77fd7ffd249003a (DEFAULT_TABLE table=default.Table_150226bfe577411d88c38a63b1af73e5 [id=34604d7af7314b66b2b9605b9bdeb724]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:01.205510   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5ea1646a2ab84345a77fd7ffd249003a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:01.213567  1851 consensus_queue.cc:1049] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:01.224236  1181 tablet_bootstrap.cc:492] T a04da9cb8f004443aa380902ea5427c5 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:01.228973  1821 tablet.cc:1867] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234: Skipping requested alter to schema version 2, tablet already version 2
I20261120 03:01:01.245580  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1faba22e8e38418383b86e42e904a761" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:01.247592   502 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_643328a529cb4231a409460bdae21438"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:01:01.248103  1857 log.cc:927] Time spent T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234: Append to log took a long time: real 0.070s	user 0.000s	sys 0.000s
W20261120 03:01:01.249094   502 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_643328a529cb4231a409460bdae21438 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:01:01.098980  1850 rpcz_store.cc:276] 1120 03:00:59.898464 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.899481 (+  1017us) service_pool.cc:224] Handling call
1120 03:01:01.098576 (+1199095us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:59.959785 (+     0us) alter_schema_op.cc:104] PREPARE ALTER-SCHEMA: Starting
1120 03:00:59.960146 (+   361us) alter_schema_op.cc:63] Acquiring schema lock in exclusive mode
1120 03:01:00.422813 (+462667us) alter_schema_op.cc:66] Acquired schema lock
1120 03:01:00.422863 (+    50us) alter_schema_op.cc:119] PREPARE ALTER-SCHEMA: finished
1120 03:01:00.422972 (+   109us) alter_schema_op.cc:128] START. Timestamp: 1795143660422888
1120 03:01:00.422986 (+    14us) op_driver.cc:350] REPLICATION: starting
1120 03:01:00.423461 (+   475us) log.cc:844] Serialized 173 byte log entry
1120 03:01:00.984137 (+560676us) op_driver.cc:466] REPLICATION: finished
1120 03:01:01.096801 (+112664us) alter_schema_op.cc:133] APPLY ALTER-SCHEMA: Starting
1120 03:01:01.097852 (+  1051us) log.cc:844] Serialized 94 byte log entry
1120 03:01:01.098456 (+   604us) alter_schema_op.cc:73] Released schema lock
1120 03:01:01.098484 (+    28us) alter_schema_op.cc:186] AlterSchemaCommitCallback: making alter schema visible
Metrics: {"child_traces":[["op",{"apply.queue_time_us":112468,"prepare.queue_time_us":59563,"prepare.run_cpu_time_us":12954,"prepare.run_wall_time_us":464906,"raft.queue_time_us":93,"raft.run_cpu_time_us":793,"raft.run_wall_time_us":793,"replication_time_us":561033,"spinlock_wait_cycles":8576,"threads_started":1}]]}
I20261120 03:01:01.282317  1853 raft_consensus.cc:534] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:01.282759  1853 raft_consensus.cc:556] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:01.284616  1853 leader_election.cc:290] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:01.285033  1853 raft_consensus.cc:534] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:01.285396  1853 raft_consensus.cc:556] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:01.288384   857 leader_election.cc:304] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:01:01.289906  1796 raft_consensus.cc:3024] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:01.290236  1796 raft_consensus.cc:534] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:01.290524  1796 raft_consensus.cc:3296] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:01.291185  1796 raft_consensus.cc:556] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:01.292047  1747 consensus_queue.cc:1049] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:01.293640   501 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 9e42a416be884341994d2d5395286798
I20261120 03:01:01.295017   501 catalog_manager.cc:6952] ReplaceTablet: tablet 9e42a416be884341994d2d5395286798 deleted and replaced by tablet ac349a09a0524b2d87289c3dd53adf6a
I20261120 03:01:01.297132   895 tablet_service.cc:1542] Processing DeleteTablet for tablet 9e42a416be884341994d2d5395286798 with delete_type TABLET_DATA_DELETED (replaced by tablet ac349a09a0524b2d87289c3dd53adf6a) from {username='slave'} at 127.0.0.1:44896
I20261120 03:01:01.299050  1853 leader_election.cc:290] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:00.871354  1699 rpcz_store.cc:276] 1120 03:00:58.976707 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:58.976843 (+   136us) service_pool.cc:224] Handling call
1120 03:00:59.799176 (+822333us) spinlock_profiling.cc:245] Waited 18.6 ms on lock 0x7b5800001200. stack: 00007fbcfbea4023 00007fbcfbea3f20 00007fbcfba3f574 00007fbd0565c964 00007fbcfbd60591 00007fbcfbd6b49a 00007fbcfbd69de9 00007fbd03065865 00007fbd03063d04 00007fbd040e8199 00007fbd02733d34 00007fbd02733cc2 00007fbd02733bf1 00007fbd02733b6c 00007fbd02732e22 00007fbd00d1e0bc
1120 03:00:59.799349 (+   173us) raft_consensus.cc:1536] Updating replica for 181 ops
1120 03:00:59.835911 (+ 36562us) raft_consensus.cc:1602] Early marking committed up to index 0
1120 03:00:59.835957 (+    46us) raft_consensus.cc:1607] Triggering prepare for 181 ops
1120 03:00:59.837632 (+  1675us) write_op.cc:270] Start()
1120 03:00:59.837759 (+   127us) write_op.cc:276] Timestamp: P: 1795143647149385 usec, L: 0
1120 03:00:59.857345 (+ 19586us) write_op.cc:270] Start()
1120 03:00:59.857441 (+    96us) write_op.cc:276] Timestamp: P: 1795143647172927 usec, L: 0
1120 03:00:59.858276 (+   835us) write_op.cc:270] Start()
1120 03:00:59.858417 (+   141us) write_op.cc:276] Timestamp: P: 1795143647180667 usec, L: 0
1120 03:00:59.859011 (+   594us) write_op.cc:270] Start()
1120 03:00:59.859094 (+    83us) write_op.cc:276] Timestamp: P: 1795143647184572 usec, L: 0
1120 03:00:59.859567 (+   473us) write_op.cc:270] Start()
1120 03:00:59.859649 (+    82us) write_op.cc:276] Timestamp: P: 1795143647189010 usec, L: 0
1120 03:00:59.867538 (+  7889us) write_op.cc:270] Start()
1120 03:00:59.867646 (+   108us) write_op.cc:276] Timestamp: P: 1795143647257963 usec, L: 0
1120 03:00:59.868192 (+   546us) write_op.cc:270] Start()
1120 03:00:59.868274 (+    82us) write_op.cc:276] Timestamp: P: 1795143647337378 usec, L: 0
1120 03:00:59.868728 (+   454us) write_op.cc:270] Start()
1120 03:00:59.868805 (+    77us) write_op.cc:276] Timestamp: P: 1795143647348246 usec, L: 0
1120 03:00:59.890392 (+ 21587us) write_op.cc:270] Start()
1120 03:00:59.890520 (+   128us) write_op.cc:276] Timestamp: P: 1795143647362917 usec, L: 0
1120 03:00:59.891123 (+   603us) write_op.cc:270] Start()
1120 03:00:59.891221 (+    98us) write_op.cc:276] Timestamp: P: 1795143647373933 usec, L: 0
1120 03:00:59.891699 (+   478us) write_op.cc:270] Start()
1120 03:00:59.891778 (+    79us) write_op.cc:276] Timestamp: P: 1795143647377057 usec, L: 0
1120 03:00:59.892235 (+   457us) write_op.cc:270] Start()
1120 03:00:59.892336 (+   101us) write_op.cc:276] Timestamp: P: 1795143647393830 usec, L: 0
1120 03:00:59.892821 (+   485us) write_op.cc:270] Start()
1120 03:00:59.892906 (+    85us) write_op.cc:276] Timestamp: P: 1795143647430779 usec, L: 0
1120 03:00:59.893364 (+   458us) write_op.cc:270] Start()
1120 03:00:59.893439 (+    75us) write_op.cc:276] Timestamp: P: 1795143647458539 usec, L: 0
1120 03:00:59.893883 (+   444us) write_op.cc:270] Start()
1120 03:00:59.893957 (+    74us) write_op.cc:276] Timestamp: P: 1795143647472185 usec, L: 0
1120 03:00:59.901797 (+  7840us) write_op.cc:270] Start()
1120 03:00:59.901894 (+    97us) write_op.cc:276] Timestamp: P: 1795143647476064 usec, L: 0
1120 03:00:59.902562 (+   668us) write_op.cc:270] Start()
1120 03:00:59.902648 (+    86us) write_op.cc:276] Timestamp: P: 1795143647514942 usec, L: 0
1120 03:00:59.903124 (+   476us) write_op.cc:270] Start()
1120 03:00:59.903216 (+    92us) write_op.cc:276] Timestamp: P: 1795143647573012 usec, L: 0
1120 03:00:59.903681 (+   465us) write_op.cc:270] Start()
1120 03:00:59.903761 (+    80us) write_op.cc:276] Timestamp: P: 1795143647578596 usec, L: 0
1120 03:00:59.904231 (+   470us) write_op.cc:270] Start()
1120 03:00:59.904303 (+    72us) write_op.cc:276] Timestamp: P: 1795143647614640 usec, L: 0
1120 03:00:59.905174 (+   871us) write_op.cc:270] Start()
1120 03:00:59.905386 (+   212us) write_op.cc:276] Timestamp: P: 1795143647625569 usec, L: 0
1120 03:00:59.905871 (+   485us) write_op.cc:270] Start()
1120 03:00:59.905947 (+    76us) write_op.cc:276] Timestamp: P: 1795143647631728 usec, L: 0
1120 03:00:59.914436 (+  8489us) write_op.cc:270] Start()
1120 03:00:59.914528 (+    92us) write_op.cc:276] Timestamp: P: 1795143647678779 usec, L: 0
1120 03:00:59.915020 (+   492us) write_op.cc:270] Start()
1120 03:00:59.915149 (+   129us) write_op.cc:276] Timestamp: P: 1795143647723724 usec, L: 0
1120 03:00:59.915701 (+   552us) write_op.cc:270] Start()
1120 03:00:59.915778 (+    77us) write_op.cc:276] Timestamp: P: 1795143647743557 usec, L: 0
1120 03:00:59.916241 (+   463us) write_op.cc:270] Start()
1120 03:00:59.916316 (+    75us) write_op.cc:276] Timestamp: P: 1795143647793415 usec, L: 0
1120 03:00:59.922856 (+  6540us) write_op.cc:270] Start()
1120 03:00:59.922941 (+    85us) write_op.cc:276] Timestamp: P: 1795143647796000 usec, L: 0
1120 03:00:59.923468 (+   527us) write_op.cc:270] Start()
1120 03:00:59.923548 (+    80us) write_op.cc:276] Timestamp: P: 1795143647861245 usec, L: 0
1120 03:00:59.924023 (+   475us) write_op.cc:270] Start()
1120 03:00:59.924123 (+   100us) write_op.cc:276] Timestamp: P: 1795143647869663 usec, L: 0
1120 03:00:59.924579 (+   456us) write_op.cc:270] Start()
1120 03:00:59.924654 (+    75us) write_op.cc:276] Timestamp: P: 1795143647890580 usec, L: 0
1120 03:00:59.925092 (+   438us) write_op.cc:270] Start()
1120 03:00:59.925175 (+    83us) write_op.cc:276] Timestamp: P: 1795143647895727 usec, L: 0
1120 03:00:59.925630 (+   455us) write_op.cc:270] Start()
1120 03:00:59.925705 (+    75us) write_op.cc:276] Timestamp: P: 1795143648040703 usec, L: 0
1120 03:00:59.926301 (+   596us) write_op.cc:270] Start()
1120 03:00:59.926378 (+    77us) write_op.cc:276] Timestamp: P: 1795143648050217 usec, L: 0
1120 03:00:59.926828 (+   450us) write_op.cc:270] Start()
1120 03:00:59.926907 (+    79us) write_op.cc:276] Timestamp: P: 1795143648057613 usec, L: 0
1120 03:00:59.927367 (+   460us) write_op.cc:270] Start()
1120 03:00:59.927441 (+    74us) write_op.cc:276] Timestamp: P: 1795143648080499 usec, L: 0
1120 03:00:59.927867 (+   426us) write_op.cc:270] Start()
1120 03:00:59.927943 (+    76us) write_op.cc:276] Timestamp: P: 1795143648190799 usec, L: 0
1120 03:00:59.928388 (+   445us) write_op.cc:270] Start()
1120 03:00:59.928457 (+    69us) write_op.cc:276] Timestamp: P: 1795143648245014 usec, L: 0
1120 03:00:59.928886 (+   429us) write_op.cc:270] Start()
1120 03:00:59.928966 (+    80us) write_op.cc:276] Timestamp: P: 1795143648257900 usec, L: 0
1120 03:00:59.929447 (+   481us) write_op.cc:270] Start()
1120 03:00:59.929527 (+    80us) write_op.cc:276] Timestamp: P: 1795143648264802 usec, L: 0
1120 03:00:59.929987 (+   460us) write_op.cc:270] Start()
1120 03:00:59.930058 (+    71us) write_op.cc:276] Timestamp: P: 1795143648267549 usec, L: 0
1120 03:00:59.944095 (+ 14037us) write_op.cc:270] Start()
1120 03:00:59.944200 (+   105us) write_op.cc:276] Timestamp: P: 1795143648360937 usec, L: 0
1120 03:00:59.944672 (+   472us) write_op.cc:270] Start()
1120 03:00:59.944753 (+    81us) write_op.cc:276] Timestamp: P: 1795143648434770 usec, L: 0
1120 03:00:59.945223 (+   470us) write_op.cc:270] Start()
1120 03:00:59.945304 (+    81us) write_op.cc:276] Timestamp: P: 1795143648441864 usec, L: 0
1120 03:00:59.945746 (+   442us) write_op.cc:270] Start()
1120 03:00:59.945818 (+    72us) write_op.cc:276] Timestamp: P: 1795143648452255 usec, L: 0
1120 03:00:59.946334 (+   516us) write_op.cc:270] Start()
1120 03:00:59.946417 (+    83us) write_op.cc:276] Timestamp: P: 1795143648481433 usec, L: 0
1120 03:00:59.946913 (+   496us) write_op.cc:270] Start()
1120 03:00:59.946999 (+    86us) write_op.cc:276] Timestamp: P: 1795143648516465 usec, L: 0
1120 03:00:59.947491 (+   492us) write_op.cc:270] Start()
1120 03:00:59.947571 (+    80us) write_op.cc:276] Timestamp: P: 1795143648591745 usec, L: 0
1120 03:00:59.948017 (+   446us) write_op.cc:270] Start()
1120 03:00:59.948164 (+   147us) write_op.cc:276] Timestamp: P: 1795143648613955 usec, L: 0
1120 03:00:59.948737 (+   573us) write_op.cc:270] Start()
1120 03:00:59.948815 (+    78us) write_op.cc:276] Timestamp: P: 1795143648670919 usec, L: 0
1120 03:00:59.949282 (+   467us) write_op.cc:270] Start()
1120 03:00:59.949370 (+    88us) write_op.cc:276] Timestamp: P: 1795143648705567 usec, L: 0
1120 03:00:59.949826 (+   456us) write_op.cc:270] Start()
1120 03:00:59.949902 (+    76us) write_op.cc:276] Timestamp: P: 1795143648729043 usec, L: 0
1120 03:00:59.950369 (+   467us) write_op.cc:270] Start()
1120 03:00:59.950444 (+    75us) write_op.cc:276] Timestamp: P: 1795143648732993 usec, L: 0
1120 03:00:59.950904 (+   460us) write_op.cc:270] Start()
1120 03:00:59.950979 (+    75us) write_op.cc:276] Timestamp: P: 1795143648780895 usec, L: 0
1120 03:00:59.951464 (+   485us) write_op.cc:270] Start()
1120 03:00:59.951544 (+    80us) write_op.cc:276] Timestamp: P: 1795143648864895 usec, L: 0
1120 03:00:59.952012 (+   468us) write_op.cc:270] Start()
1120 03:00:59.952088 (+    76us) write_op.cc:276] Timestamp: P: 1795143648869785 usec, L: 0
1120 03:00:59.952535 (+   447us) write_op.cc:270] Start()
1120 03:00:59.952609 (+    74us) write_op.cc:276] Timestamp: P: 1795143649030985 usec, L: 0
1120 03:00:59.953163 (+   554us) write_op.cc:270] Start()
1120 03:00:59.953249 (+    86us) write_op.cc:276] Timestamp: P: 1795143649034113 usec, L: 0
1120 03:00:59.966398 (+ 13149us) write_op.cc:270] Start()
1120 03:00:59.966490 (+    92us) write_op.cc:276] Timestamp: P: 1795143649037702 usec, L: 0
1120 03:00:59.967005 (+   515us) write_op.cc:270] Start()
1120 03:00:59.967083 (+    78us) write_op.cc:276] Timestamp: P: 1795143649091980 usec, L: 0
1120 03:00:59.967555 (+   472us) write_op.cc:270] Start()
1120 03:00:59.967629 (+    74us) write_op.cc:276] Timestamp: P: 1795143649093903 usec, L: 0
1120 03:01:00.018909 (+ 51280us) write_op.cc:270] Start()
1120 03:01:00.019005 (+    96us) write_op.cc:276] Timestamp: P: 1795143649140453 usec, L: 0
1120 03:01:00.019547 (+   542us) write_op.cc:270] Start()
1120 03:01:00.019620 (+    73us) write_op.cc:276] Timestamp: P: 1795143649227838 usec, L: 0
1120 03:01:00.020095 (+   475us) write_op.cc:270] Start()
1120 03:01:00.020174 (+    79us) write_op.cc:276] Timestamp: P: 1795143649238099 usec, L: 0
1120 03:01:00.020597 (+   423us) write_op.cc:270] Start()
1120 03:01:00.020667 (+    70us) write_op.cc:276] Timestamp: P: 1795143649322928 usec, L: 0
1120 03:01:00.031546 (+ 10879us) write_op.cc:270] Start()
1120 03:01:00.031638 (+    92us) write_op.cc:276] Timestamp: P: 1795143649331732 usec, L: 0
1120 03:01:00.032146 (+   508us) write_op.cc:270] Start()
1120 03:01:00.032219 (+    73us) write_op.cc:276] Timestamp: P: 1795143649343380 usec, L: 0
1120 03:01:00.032670 (+   451us) write_op.cc:270] Start()
1120 03:01:00.032752 (+    82us) write_op.cc:276] Timestamp: P: 1795143649377865 usec, L: 0
1120 03:01:00.033197 (+   445us) write_op.cc:270] Start()
1120 03:01:00.033277 (+    80us) write_op.cc:276] Timestamp: P: 1795143649477872 usec, L: 0
1120 03:01:00.033757 (+   480us) write_op.cc:270] Start()
1120 03:01:00.033837 (+    80us) write_op.cc:276] Timestamp: P: 1795143649513916 usec, L: 0
1120 03:01:00.048957 (+ 15120us) write_op.cc:270] Start()
1120 03:01:00.049079 (+   122us) write_op.cc:276] Timestamp: P: 1795143649620627 usec, L: 0
1120 03:01:00.049646 (+   567us) write_op.cc:270] Start()
1120 03:01:00.049746 (+   100us) write_op.cc:276] Timestamp: P: 1795143649706078 usec, L: 0
1120 03:01:00.050249 (+   503us) write_op.cc:270] Start()
1120 03:01:00.050318 (+    69us) write_op.cc:276] Timestamp: P: 1795143649709944 usec, L: 0
1120 03:01:00.050765 (+   447us) write_op.cc:270] Start()
1120 03:01:00.050839 (+    74us) write_op.cc:276] Timestamp: P: 1795143649906271 usec, L: 0
1120 03:01:00.051262 (+   423us) write_op.cc:270] Start()
1120 03:01:00.051327 (+    65us) write_op.cc:276] Timestamp: P: 1795143649909696 usec, L: 0
1120 03:01:00.051787 (+   460us) write_op.cc:270] Start()
1120 03:01:00.051860 (+    73us) write_op.cc:276] Timestamp: P: 1795143649919265 usec, L: 0
1120 03:01:00.052332 (+   472us) write_op.cc:270] Start()
1120 03:01:00.052405 (+    73us) write_op.cc:276] Timestamp: P: 1795143650055332 usec, L: 0
1120 03:01:00.052904 (+   499us) write_op.cc:270] Start()
1120 03:01:00.052987 (+    83us) write_op.cc:276] Timestamp: P: 1795143650248546 usec, L: 0
1120 03:01:00.053467 (+   480us) write_op.cc:270] Start()
1120 03:01:00.053539 (+    72us) write_op.cc:276] Timestamp: P: 1795143650316789 usec, L: 0
1120 03:01:00.053968 (+   429us) write_op.cc:270] Start()
1120 03:01:00.054040 (+    72us) write_op.cc:276] Timestamp: P: 1795143650320612 usec, L: 0
1120 03:01:00.130215 (+ 76175us) write_op.cc:270] Start()
1120 03:01:00.130323 (+   108us) write_op.cc:276] Timestamp: P: 1795143650354711 usec, L: 0
1120 03:01:00.130923 (+   600us) write_op.cc:270] Start()
1120 03:01:00.131008 (+    85us) write_op.cc:276] Timestamp: P: 1795143650474913 usec, L: 0
1120 03:01:00.131515 (+   507us) write_op.cc:270] Start()
1120 03:01:00.131594 (+    79us) write_op.cc:276] Timestamp: P: 1795143650571420 usec, L: 0
1120 03:01:00.132088 (+   494us) write_op.cc:270] Start()
1120 03:01:00.132161 (+    73us) write_op.cc:276] Timestamp: P: 1795143650647496 usec, L: 0
1120 03:01:00.132616 (+   455us) write_op.cc:270] Start()
1120 03:01:00.132704 (+    88us) write_op.cc:276] Timestamp: P: 1795143650811656 usec, L: 0
1120 03:01:00.133201 (+   497us) write_op.cc:270] Start()
1120 03:01:00.133295 (+    94us) write_op.cc:276] Timestamp: P: 1795143650816038 usec, L: 0
1120 03:01:00.133773 (+   478us) write_op.cc:270] Start()
1120 03:01:00.133847 (+    74us) write_op.cc:276] Timestamp: P: 1795143650946084 usec, L: 0
1120 03:01:00.152938 (+ 19091us) write_op.cc:270] Start()
1120 03:01:00.153035 (+    97us) write_op.cc:276] Timestamp: P: 1795143650988934 usec, L: 0
1120 03:01:00.153588 (+   553us) write_op.cc:270] Start()
1120 03:01:00.153667 (+    79us) write_op.cc:276] Timestamp: P: 1795143651010663 usec, L: 0
1120 03:01:00.154177 (+   510us) write_op.cc:270] Start()
1120 03:01:00.154268 (+    91us) write_op.cc:276] Timestamp: P: 1795143651030156 usec, L: 0
1120 03:01:00.154801 (+   533us) write_op.cc:270] Start()
1120 03:01:00.154877 (+    76us) write_op.cc:276] Timestamp: P: 1795143651075487 usec, L: 0
1120 03:01:00.155326 (+   449us) write_op.cc:270] Start()
1120 03:01:00.155408 (+    82us) write_op.cc:276] Timestamp: P: 1795143651204614 usec, L: 0
1120 03:01:00.165553 (+ 10145us) write_op.cc:270] Start()
1120 03:01:00.165760 (+   207us) write_op.cc:276] Timestamp: P: 1795143651276569 usec, L: 0
1120 03:01:00.166320 (+   560us) write_op.cc:270] Start()
1120 03:01:00.166413 (+    93us) write_op.cc:276] Timestamp: P: 1795143651302662 usec, L: 0
1120 03:01:00.166937 (+   524us) write_op.cc:270] Start()
1120 03:01:00.167012 (+    75us) write_op.cc:276] Timestamp: P: 1795143651342452 usec, L: 0
1120 03:01:00.167496 (+   484us) write_op.cc:270] Start()
1120 03:01:00.167575 (+    79us) write_op.cc:276] Timestamp: P: 1795143651419327 usec, L: 0
1120 03:01:00.168575 (+  1000us) write_op.cc:270] Start()
1120 03:01:00.168653 (+    78us) write_op.cc:276] Timestamp: P: 1795143651464492 usec, L: 0
1120 03:01:00.193245 (+ 24592us) write_op.cc:270] Start()
1120 03:01:00.193337 (+    92us) write_op.cc:276] Timestamp: P: 1795143651497040 usec, L: 0
1120 03:01:00.198595 (+  5258us) write_op.cc:270] Start()
1120 03:01:00.198844 (+   249us) write_op.cc:276] Timestamp: P: 1795143651552243 usec, L: 0
1120 03:01:00.199459 (+   615us) write_op.cc:270] Start()
1120 03:01:00.199534 (+    75us) write_op.cc:276] Timestamp: P: 1795143651577008 usec, L: 0
1120 03:01:00.200004 (+   470us) write_op.cc:270] Start()
1120 03:01:00.200079 (+    75us) write_op.cc:276] Timestamp: P: 1795143651651204 usec, L: 0
1120 03:01:00.200533 (+   454us) write_op.cc:270] Start()
1120 03:01:00.200606 (+    73us) write_op.cc:276] Timestamp: P: 1795143651682394 usec, L: 0
1120 03:01:00.203426 (+  2820us) write_op.cc:270] Start()
1120 03:01:00.203508 (+    82us) write_op.cc:276] Timestamp: P: 1795143651684939 usec, L: 0
1120 03:01:00.203977 (+   469us) write_op.cc:270] Start()
1120 03:01:00.204075 (+    98us) write_op.cc:276] Timestamp: P: 1795143651704813 usec, L: 0
1120 03:01:00.204567 (+   492us) write_op.cc:270] Start()
1120 03:01:00.204644 (+    77us) write_op.cc:276] Timestamp: P: 1795143651744345 usec, L: 0
1120 03:01:00.205152 (+   508us) write_op.cc:270] Start()
1120 03:01:00.205250 (+    98us) write_op.cc:276] Timestamp: P: 1795143651800312 usec, L: 0
1120 03:01:00.205709 (+   459us) write_op.cc:270] Start()
1120 03:01:00.205786 (+    77us) write_op.cc:276] Timestamp: P: 1795143651880915 usec, L: 0
1120 03:01:00.216369 (+ 10583us) write_op.cc:270] Start()
1120 03:01:00.216465 (+    96us) write_op.cc:276] Timestamp: P: 1795143651919576 usec, L: 0
1120 03:01:00.217082 (+   617us) write_op.cc:270] Start()
1120 03:01:00.218409 (+  1327us) write_op.cc:276] Timestamp: P: 1795143651952001 usec, L: 0
1120 03:01:00.218946 (+   537us) write_op.cc:270] Start()
1120 03:01:00.219027 (+    81us) write_op.cc:276] Timestamp: P: 1795143652007397 usec, L: 0
1120 03:01:00.219487 (+   460us) write_op.cc:270] Start()
1120 03:01:00.219563 (+    76us) write_op.cc:276] Timestamp: P: 1795143652048051 usec, L: 0
1120 03:01:00.220023 (+   460us) write_op.cc:270] Start()
1120 03:01:00.220099 (+    76us) write_op.cc:276] Timestamp: P: 1795143652119342 usec, L: 0
1120 03:01:00.220540 (+   441us) write_op.cc:270] Start()
1120 03:01:00.223278 (+  2738us) write_op.cc:276] Timestamp: P: 1795143652165282 usec, L: 0
1120 03:01:00.223810 (+   532us) write_op.cc:270] Start()
1120 03:01:00.223893 (+    83us) write_op.cc:276] Timestamp: P: 1795143652168107 usec, L: 0
1120 03:01:00.224360 (+   467us) write_op.cc:270] Start()
1120 03:01:00.224439 (+    79us) write_op.cc:276] Timestamp: P: 1795143652193304 usec, L: 0
1120 03:01:00.224916 (+   477us) write_op.cc:270] Start()
1120 03:01:00.224990 (+    74us) write_op.cc:276] Timestamp: P: 1795143652249475 usec, L: 0
1120 03:01:00.225438 (+   448us) write_op.cc:270] Start()
1120 03:01:00.225512 (+    74us) write_op.cc:276] Timestamp: P: 1795143652253938 usec, L: 0
1120 03:01:00.226013 (+   501us) write_op.cc:270] Start()
1120 03:01:00.258105 (+ 32092us) write_op.cc:276] Timestamp: P: 1795143652287365 usec, L: 0
1120 03:01:00.258720 (+   615us) write_op.cc:270] Start()
1120 03:01:00.258805 (+    85us) write_op.cc:276] Timestamp: P: 1795143652352121 usec, L: 0
1120 03:01:00.259324 (+   519us) write_op.cc:270] Start()
1120 03:01:00.259403 (+    79us) write_op.cc:276] Timestamp: P: 1795143652411345 usec, L: 0
1120 03:01:00.259887 (+   484us) write_op.cc:270] Start()
1120 03:01:00.259971 (+    84us) write_op.cc:276] Timestamp: P: 1795143652423320 usec, L: 0
1120 03:01:00.260469 (+   498us) write_op.cc:270] Start()
1120 03:01:00.262460 (+  1991us) write_op.cc:276] Timestamp: P: 1795143652427479 usec, L: 0
1120 03:01:00.262971 (+   511us) write_op.cc:270] Start()
1120 03:01:00.263051 (+    80us) write_op.cc:276] Timestamp: P: 1795143652494999 usec, L: 0
1120 03:01:00.263505 (+   454us) write_op.cc:270] Start()
1120 03:01:00.263579 (+    74us) write_op.cc:276] Timestamp: P: 1795143652591524 usec, L: 0
1120 03:01:00.263998 (+   419us) write_op.cc:270] Start()
1120 03:01:00.264085 (+    87us) write_op.cc:276] Timestamp: P: 1795143652594503 usec, L: 0
1120 03:01:00.264539 (+   454us) write_op.cc:270] Start()
1120 03:01:00.264624 (+    85us) write_op.cc:276] Timestamp: P: 1795143652607864 usec, L: 0
1120 03:01:00.265070 (+   446us) write_op.cc:270] Start()
1120 03:01:00.265143 (+    73us) write_op.cc:276] Timestamp: P: 1795143652630681 usec, L: 0
1120 03:01:00.265607 (+   464us) write_op.cc:270] Start()
1120 03:01:00.265680 (+    73us) write_op.cc:276] Timestamp: P: 1795143652712194 usec, L: 0
1120 03:01:00.316875 (+ 51195us) write_op.cc:270] Start()
1120 03:01:00.347374 (+ 30499us) write_op.cc:276] Timestamp: P: 1795143652744077 usec, L: 0
1120 03:01:00.350607 (+  3233us) write_op.cc:270] Start()
1120 03:01:00.350710 (+   103us) write_op.cc:276] Timestamp: P: 1795143652746851 usec, L: 0
1120 03:01:00.351257 (+   547us) write_op.cc:270] Start()
1120 03:01:00.351335 (+    78us) write_op.cc:276] Timestamp: P: 1795143652756627 usec, L: 0
1120 03:01:00.351809 (+   474us) write_op.cc:270] Start()
1120 03:01:00.351882 (+    73us) write_op.cc:276] Timestamp: P: 1795143652875803 usec, L: 0
1120 03:01:00.352343 (+   461us) write_op.cc:270] Start()
1120 03:01:00.352417 (+    74us) write_op.cc:276] Timestamp: P: 1795143652941061 usec, L: 0
1120 03:01:00.352889 (+   472us) write_op.cc:270] Start()
1120 03:01:00.352976 (+    87us) write_op.cc:276] Timestamp: P: 1795143652946560 usec, L: 0
1120 03:01:00.353464 (+   488us) write_op.cc:270] Start()
1120 03:01:00.353538 (+    74us) write_op.cc:276] Timestamp: P: 1795143653007228 usec, L: 0
1120 03:01:00.354008 (+   470us) write_op.cc:270] Start()
1120 03:01:00.363965 (+  9957us) write_op.cc:276] Timestamp: P: 1795143653021239 usec, L: 0
1120 03:01:00.364677 (+   712us) write_op.cc:270] Start()
1120 03:01:00.364788 (+   111us) write_op.cc:276] Timestamp: P: 1795143653081535 usec, L: 0
1120 03:01:00.379725 (+ 14937us) write_op.cc:270] Start()
1120 03:01:00.379834 (+   109us) write_op.cc:276] Timestamp: P: 1795143653155964 usec, L: 0
1120 03:01:00.380346 (+   512us) write_op.cc:270] Start()
1120 03:01:00.380425 (+    79us) write_op.cc:276] Timestamp: P: 1795143653234639 usec, L: 0
1120 03:01:00.380912 (+   487us) write_op.cc:270] Start()
1120 03:01:00.380991 (+    79us) write_op.cc:276] Timestamp: P: 1795143653237792 usec, L: 0
1120 03:01:00.381446 (+   455us) write_op.cc:270] Start()
1120 03:01:00.381525 (+    79us) write_op.cc:276] Timestamp: P: 1795143653240535 usec, L: 0
1120 03:01:00.382018 (+   493us) write_op.cc:270] Start()
1120 03:01:00.389144 (+  7126us) write_op.cc:276] Timestamp: P: 1795143653300265 usec, L: 0
1120 03:01:00.389794 (+   650us) write_op.cc:270] Start()
1120 03:01:00.389886 (+    92us) write_op.cc:276] Timestamp: P: 1795143653471280 usec, L: 0
1120 03:01:00.391720 (+  1834us) write_op.cc:270] Start()
1120 03:01:00.391814 (+    94us) write_op.cc:276] Timestamp: P: 1795143653473290 usec, L: 0
1120 03:01:00.392319 (+   505us) write_op.cc:270] Start()
1120 03:01:00.392397 (+    78us) write_op.cc:276] Timestamp: P: 1795143653547452 usec, L: 0
1120 03:01:00.393443 (+  1046us) write_op.cc:270] Start()
1120 03:01:00.393515 (+    72us) write_op.cc:276] Timestamp: P: 1795143653549511 usec, L: 0
1120 03:01:00.393988 (+   473us) write_op.cc:270] Start()
1120 03:01:00.394057 (+    69us) write_op.cc:276] Timestamp: P: 1795143653609557 usec, L: 0
1120 03:01:00.394529 (+   472us) write_op.cc:270] Start()
1120 03:01:00.394600 (+    71us) write_op.cc:276] Timestamp: P: 1795143653619807 usec, L: 0
1120 03:01:00.395041 (+   441us) write_op.cc:270] Start()
1120 03:01:00.395111 (+    70us) write_op.cc:276] Timestamp: P: 1795143653751042 usec, L: 0
1120 03:01:00.395625 (+   514us) write_op.cc:270] Start()
1120 03:01:00.395706 (+    81us) write_op.cc:276] Timestamp: P: 1795143653841984 usec, L: 0
1120 03:01:00.396120 (+   414us) write_op.cc:270] Start()
1120 03:01:00.396189 (+    69us) write_op.cc:276] Timestamp: P: 1795143654109867 usec, L: 0
1120 03:01:00.396637 (+   448us) write_op.cc:270] Start()
1120 03:01:00.396736 (+    99us) write_op.cc:276] Timestamp: P: 1795143654112948 usec, L: 0
1120 03:01:00.397203 (+   467us) write_op.cc:270] Start()
1120 03:01:00.397271 (+    68us) write_op.cc:276] Timestamp: P: 1795143654175182 usec, L: 0
1120 03:01:00.397695 (+   424us) write_op.cc:270] Start()
1120 03:01:00.397760 (+    65us) write_op.cc:276] Timestamp: P: 1795143654489196 usec, L: 0
1120 03:01:00.438293 (+ 40533us) write_op.cc:270] Start()
1120 03:01:00.438415 (+   122us) write_op.cc:276] Timestamp: P: 1795143654493143 usec, L: 0
1120 03:01:00.439497 (+  1082us) write_op.cc:270] Start()
1120 03:01:00.439591 (+    94us) write_op.cc:276] Timestamp: P: 1795143654497949 usec, L: 0
1120 03:01:00.440114 (+   523us) write_op.cc:270] Start()
1120 03:01:00.440181 (+    67us) write_op.cc:276] Timestamp: P: 1795143654641957 usec, L: 0
1120 03:01:00.440631 (+   450us) write_op.cc:270] Start()
1120 03:01:00.440738 (+   107us) write_op.cc:276] Timestamp: P: 1795143654684509 usec, L: 0
1120 03:01:00.441156 (+   418us) write_op.cc:270] Start()
1120 03:01:00.441222 (+    66us) write_op.cc:276] Timestamp: P: 1795143654712173 usec, L: 0
1120 03:01:00.450438 (+  9216us) write_op.cc:270] Start()
1120 03:01:00.450520 (+    82us) write_op.cc:276] Timestamp: P: 1795143654823370 usec, L: 0
1120 03:01:00.451025 (+   505us) write_op.cc:270] Start()
1120 03:01:00.451090 (+    65us) write_op.cc:276] Timestamp: P: 1795143654856207 usec, L: 0
1120 03:01:00.451544 (+   454us) write_op.cc:270] Start()
1120 03:01:00.451611 (+    67us) write_op.cc:276] Timestamp: P: 1795143654923254 usec, L: 0
1120 03:01:00.460225 (+  8614us) write_op.cc:270] Start()
1120 03:01:00.461016 (+   791us) write_op.cc:276] Timestamp: P: 1795143654944212 usec, L: 0
1120 03:01:00.461857 (+   841us) write_op.cc:270] Start()
1120 03:01:00.462030 (+   173us) write_op.cc:276] Timestamp: P: 1795143655010479 usec, L: 0
1120 03:01:00.462546 (+   516us) write_op.cc:270] Start()
1120 03:01:00.462616 (+    70us) write_op.cc:276] Timestamp: P: 1795143655103800 usec, L: 0
1120 03:01:00.463096 (+   480us) write_op.cc:270] Start()
1120 03:01:00.463167 (+    71us) write_op.cc:276] Timestamp: P: 1795143655339680 usec, L: 0
1120 03:01:00.463605 (+   438us) write_op.cc:270] Start()
1120 03:01:00.463674 (+    69us) write_op.cc:276] Timestamp: P: 1795143656123442 usec, L: 0
1120 03:01:00.464196 (+   522us) write_op.cc:270] Start()
1120 03:01:00.464271 (+    75us) write_op.cc:276] Timestamp: P: 1795143656125491 usec, L: 0
1120 03:01:00.464746 (+   475us) write_op.cc:270] Start()
1120 03:01:00.464813 (+    67us) write_op.cc:276] Timestamp: P: 1795143656184102 usec, L: 0
1120 03:01:00.465249 (+   436us) write_op.cc:270] Start()
1120 03:01:00.465322 (+    73us) write_op.cc:276] Timestamp: P: 1795143656211521 usec, L: 0
1120 03:01:00.465746 (+   424us) write_op.cc:270] Start()
1120 03:01:00.465841 (+    95us) write_op.cc:276] Timestamp: P: 1795143656236804 usec, L: 0
1120 03:01:00.469535 (+  3694us) write_op.cc:270] Start()
1120 03:01:00.469621 (+    86us) write_op.cc:276] Timestamp: P: 1795143656838103 usec, L: 0
1120 03:01:00.472773 (+  3152us) write_op.cc:270] Start()
1120 03:01:00.472852 (+    79us) write_op.cc:276] Timestamp: P: 1795143657199936 usec, L: 0
1120 03:01:00.473319 (+   467us) write_op.cc:270] Start()
1120 03:01:00.473393 (+    74us) write_op.cc:276] Timestamp: P: 1795143657203132 usec, L: 0
1120 03:01:00.473827 (+   434us) write_op.cc:270] Start()
1120 03:01:00.473897 (+    70us) write_op.cc:276] Timestamp: P: 1795143657305507 usec, L: 0
1120 03:01:00.482537 (+  8640us) write_op.cc:270] Start()
1120 03:01:00.482630 (+    93us) write_op.cc:276] Timestamp: P: 1795143657509446 usec, L: 0
1120 03:01:00.483191 (+   561us) write_op.cc:270] Start()
1120 03:01:00.483267 (+    76us) write_op.cc:276] Timestamp: P: 1795143657718694 usec, L: 0
1120 03:01:00.483734 (+   467us) write_op.cc:270] Start()
1120 03:01:00.483813 (+    79us) write_op.cc:276] Timestamp: P: 1795143657782861 usec, L: 0
1120 03:01:00.490776 (+  6963us) write_op.cc:270] Start()
1120 03:01:00.490871 (+    95us) write_op.cc:276] Timestamp: P: 1795143657945245 usec, L: 0
1120 03:01:00.492898 (+  2027us) write_op.cc:270] Start()
1120 03:01:00.492990 (+    92us) write_op.cc:276] Timestamp: P: 1795143658111042 usec, L: 0
1120 03:01:00.493478 (+   488us) write_op.cc:270] Start()
1120 03:01:00.493547 (+    69us) write_op.cc:276] Timestamp: P: 1795143658145037 usec, L: 0
1120 03:01:00.494011 (+   464us) write_op.cc:270] Start()
1120 03:01:00.494104 (+    93us) write_op.cc:276] Timestamp: P: 1795143658194399 usec, L: 0
1120 03:01:00.569357 (+ 75253us) log.cc:844] Serialized 108806 byte log entry
1120 03:01:00.625227 (+ 55870us) raft_consensus.cc:1713] Marking committed up to 190
1120 03:01:00.657417 (+ 32190us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:00.657439 (+    22us) write_op.cc:459] Released schema lock
1120 03:01:00.665766 (+  8327us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:00.665786 (+    20us) write_op.cc:459] Released schema lock
1120 03:01:00.834622 (+168836us) raft_consensus.cc:1765] Filling consensus response to leader.
1120 03:01:00.836563 (+  1941us) spinlock_profiling.cc:245] Waited 1 s on lock 0x7b5400040454. stack: 00007fbcfbea4023 00007fbcfbea3f20 00007fbcfba3f574 00007fbd0565c964 00007fbd0565c8f9 00007fbd02fcd510 00007fbd0306e14a 00007fbd04f3af5c 00007fbd04f4356c 00007fbd04f43518 00007fbd04f43498 00007fbd04f43453 00007fbd04f426fc 00007fbd034e82c6 00007fbd034e8258 00007fbd0308618a
1120 03:01:00.870184 (+ 33621us) raft_consensus.cc:1736] Waiting on the replicates to finish logging
1120 03:01:00.870259 (+    75us) raft_consensus.cc:1753] finished
1120 03:01:00.870291 (+    32us) raft_consensus.cc:1759] UpdateReplicas() finished
1120 03:01:00.870824 (+   533us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:00:59.855193 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:00:59.855763 (+   570us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:00:59.855799 (+    36us) write_op.cc:440] Acquired schema lock
1120 03:00:59.855822 (+    23us) tablet.cc:710] Decoding operations
1120 03:00:59.856237 (+   415us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:00:59.856309 (+    72us) write_op.cc:646] Partition lock acquired for write op
1120 03:00:59.856335 (+    26us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:00:59.856560 (+   225us) tablet.cc:758] Row locks acquired
1120 03:00:59.856578 (+    18u
I20261120 03:01:01.304071  1796 leader_election.cc:290] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:01.307361   756 tablet_service.cc:1542] Processing DeleteTablet for tablet 9e42a416be884341994d2d5395286798 with delete_type TABLET_DATA_DELETED (replaced by tablet ac349a09a0524b2d87289c3dd53adf6a) from {username='slave'} at 127.0.0.1:52644
I20261120 03:01:01.309592  1051 tablet_service.cc:1542] Processing DeleteTablet for tablet 9e42a416be884341994d2d5395286798 with delete_type TABLET_DATA_DELETED (replaced by tablet ac349a09a0524b2d87289c3dd53adf6a) from {username='slave'} at 127.0.0.1:50744
W20261120 03:01:01.314517   486 catalog_manager.cc:5061] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 9e42a416be884341994d2d5395286798 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 9e42a416be884341994d2d5395286798 already in progress: creating tablet
I20261120 03:01:01.327713  1858 sys_catalog.cc:455] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "18331bef5c884dcab9dbd26bc2c82273" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } } }
I20261120 03:01:01.329298  1858 sys_catalog.cc:458] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273 [sys.catalog]: This master's current role is: LEADER
I20261120 03:01:01.342701  1860 tablet.cc:1867] T 0db7db25d779459ebde532f938493975 P 5beeefe8bef54caeac463d05d8c30fa6: Skipping requested alter to schema version 2, tablet already version 2
I20261120 03:01:01.354709  1011 leader_election.cc:304] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440
W20261120 03:01:01.372735   487 catalog_manager.cc:5061] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 9e42a416be884341994d2d5395286798 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 9e42a416be884341994d2d5395286798 already in progress: creating tablet
W20261120 03:01:01.376684   709 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 115034us
I20261120 03:01:01.383847   498 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet d736d65c3e0b419084b4934dad83f16a (table default.Table_04ddab7f706b43e7ba12fa247b3ca933 [id=9f571a69327a498e89f323c9ac0eee34]). Expected version 1 got 0
I20261120 03:01:01.385429   498 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
I20261120 03:01:01.404888   502 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_40975331cb484f45b9592c00eab3cc54"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:01.416410  1860 tablet.cc:1872] T 1237b0d2fa414ef38271835b5d69d6b9 P 5beeefe8bef54caeac463d05d8c30fa6: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20261120 03:01:01.420300  1806 raft_consensus.cc:534] T 5da3d2724dd34aed88e6b08983188004 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:01.420763  1806 raft_consensus.cc:556] T 5da3d2724dd34aed88e6b08983188004 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:01.422887  1806 leader_election.cc:290] T 5da3d2724dd34aed88e6b08983188004 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
W20261120 03:01:01.434572   710 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 145941us
I20261120 03:01:01.456029  1679 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" instance_seqno: 1795143642520384) as {username='slave'} at 127.0.96.65:49637; Asking this server to re-register.
I20261120 03:01:01.462738  1747 raft_consensus.cc:534] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:01.463222  1747 raft_consensus.cc:556] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:01.465304  1747 leader_election.cc:290] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:01.466219  1796 raft_consensus.cc:534] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:01.466629  1796 raft_consensus.cc:556] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:01.468381  1796 leader_election.cc:290] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:01.487015  1821 tablet.cc:1872] T 1237b0d2fa414ef38271835b5d69d6b9 P 823abdbd90ba47bbacad0f8dcff78234: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20261120 03:01:01.488900  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5da3d2724dd34aed88e6b08983188004" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:01:01.514508  1011 raft_consensus.cc:2902] T a1192a7f050a4f5387a30b2216dec5f0 P 823abdbd90ba47bbacad0f8dcff78234: unable to run election callback: Service unavailable: Thread pool token was shut down
W20261120 03:01:01.518057  1011 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 163415us
I20261120 03:01:01.526821  1181 tablet_bootstrap.cc:654] T a04da9cb8f004443aa380902ea5427c5 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:01.542563  1324 tablet_bootstrap.cc:654] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
W20261120 03:01:01.547286   857 leader_election.cc:343] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:01.552846  1796 consensus_queue.cc:1049] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20261120 03:01:01.565389   502 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_40975331cb484f45b9592c00eab3cc54 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:01:01.566390   493 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
I20261120 03:01:01.567354   493 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 1faba22e8e38418383b86e42e904a761 (table default.Table_47101d55d8f6485c816a02a54fda67a8 [id=252d229ff1334cd99ec99e5c921031bb]). Expected version 1 got 0
I20261120 03:01:01.567893   493 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 0db7db25d779459ebde532f938493975 (table default.Table_9fe8b8c914534e4ca5602de7c1b1d280 [id=f3b5f7a4e5494e82a472d0025df96e0c]). Expected version 2 got 0
I20261120 03:01:01.572715   823 heartbeater.cc:462] Registering TS with master...
I20261120 03:01:01.349992  1180 tablet_bootstrap.cc:654] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:01.561226  1125 heartbeater.cc:462] Registering TS with master...
I20261120 03:01:01.590714   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5da3d2724dd34aed88e6b08983188004" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:01.599229  1850 tablet.cc:1872] T 1237b0d2fa414ef38271835b5d69d6b9 P 3d30c2428730481db9b0ffd279dd3440: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20261120 03:01:01.601821  1595 raft_consensus.cc:534] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:01.602288  1595 raft_consensus.cc:556] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:01.604147  1595 leader_election.cc:290] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:01:01.604537  1595 raft_consensus.cc:534] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:01.604899  1595 raft_consensus.cc:556] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:01.621362  1595 leader_election.cc:290] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
W20261120 03:01:01.622485   709 leader_election.cc:343] T 5da3d2724dd34aed88e6b08983188004 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:01.635408  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1faba22e8e38418383b86e42e904a761" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:01:01.636744  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "600a18b7e5104a78935d906b8adfde6f" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:01.639848   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "600a18b7e5104a78935d906b8adfde6f" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:01.640168   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1faba22e8e38418383b86e42e904a761" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:01.640549   777 raft_consensus.cc:2062] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:01:01.640609   776 raft_consensus.cc:3296] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:01.642417   771 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a04da9cb8f004443aa380902ea5427c5" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:01.642843   772 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9dadc05b726346309d6b350f9abb4abb" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:01.643103   771 raft_consensus.cc:2062] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:01:01.643553   772 raft_consensus.cc:2062] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
W20261120 03:01:01.646420   709 consensus_peers.cc:597] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1885397e42774a89bb347bc776284bc0. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:01:01.646399   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "600a18b7e5104a78935d906b8adfde6f" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:01.647056   918 raft_consensus.cc:2062] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:01:01.650367  1079 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a04da9cb8f004443aa380902ea5427c5" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:01.651448  1068 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "600a18b7e5104a78935d906b8adfde6f" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:01.645793   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a04da9cb8f004443aa380902ea5427c5" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:01.655754   919 raft_consensus.cc:2062] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:01:01.657296   857 leader_election.cc:304] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:01:01.658106   857 leader_election.cc:304] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:01:01.658749  1853 raft_consensus.cc:3024] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:01.658975  1747 raft_consensus.cc:3024] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:01.659061  1853 raft_consensus.cc:534] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:01.659334  1747 raft_consensus.cc:534] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:01.659413  1853 raft_consensus.cc:3296] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:01.659672  1747 raft_consensus.cc:3296] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:01.660184  1853 raft_consensus.cc:556] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:01.660370  1747 raft_consensus.cc:556] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:01.666411   857 leader_election.cc:304] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:01:01.667102  1837 raft_consensus.cc:3024] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:01.667430  1837 raft_consensus.cc:534] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:01.667725  1837 raft_consensus.cc:3296] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:01.671873  1747 leader_election.cc:290] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:01.639271   773 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a1192a7f050a4f5387a30b2216dec5f0" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:01.673584   773 raft_consensus.cc:2628] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 823abdbd90ba47bbacad0f8dcff78234 in this term.
I20261120 03:01:01.676182   776 raft_consensus.cc:2062] T 1faba22e8e38418383b86e42e904a761 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:01:01.679701  1181 tablet_bootstrap.cc:492] T a04da9cb8f004443aa380902ea5427c5 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:01.680119  1181 ts_tablet_manager.cc:1449] T a04da9cb8f004443aa380902ea5427c5 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.456s	user 0.010s	sys 0.000s
I20261120 03:01:01.682907  1761 ts_tablet_manager.cc:1975] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:01.683559  1761 log.cc:1199] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/020be7f7cda0467aab56f657cbe8d0ad
I20261120 03:01:01.691414  1747 raft_consensus.cc:534] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:01.692075  1747 raft_consensus.cc:556] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:01.695654  1821 tablet.cc:1872] T d736d65c3e0b419084b4934dad83f16a P 823abdbd90ba47bbacad0f8dcff78234: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20261120 03:01:01.697381  1853 leader_election.cc:290] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:01.700481  1747 leader_election.cc:290] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
W20261120 03:01:01.702885   488 catalog_manager.cc:4863] Async tablet task AlterTable RPC for tablet 8935b23c661c4c1ca7bde7c893927502 (table default.Table_92f6cc71382f4deba2472c49ae8f6a36 [id=51df6e371d3441578e57ea5597ee81a6], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 8935b23c661c4c1ca7bde7c893927502 consensus state has no leader
I20261120 03:01:01.704895   493 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet d736d65c3e0b419084b4934dad83f16a (table default.Table_04ddab7f706b43e7ba12fa247b3ca933 [id=9f571a69327a498e89f323c9ac0eee34]). Expected version 1 got 0
I20261120 03:01:01.705499   493 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 0179a59749a14168a840e9d4e610be1f (table default.Table_cc56bf0ad8344f1a94d5da6e4d77e8c0 [id=b9034a083c7e4133a746b336335d3e42]). Expected version 1 got 0
W20261120 03:01:01.725574   857 leader_election.cc:343] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Not found: Tablet not found: a1192a7f050a4f5387a30b2216dec5f0
W20261120 03:01:01.727511   857 leader_election.cc:343] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:01.638814  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a04da9cb8f004443aa380902ea5427c5" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:01.729681  1677 ts_manager.cc:210] Registered new tserver with Master: 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:01.673076  1324 tablet_bootstrap.cc:492] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:01:01.730526  1324 ts_tablet_manager.cc:1449] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.598s	user 0.004s	sys 0.006s
W20261120 03:01:01.733939   710 leader_election.cc:343] T 5da3d2724dd34aed88e6b08983188004 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:01.734691  1851 consensus_queue.cc:1049] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:01.707501  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9dadc05b726346309d6b350f9abb4abb" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:01.714867  1079 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a04da9cb8f004443aa380902ea5427c5" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:01:01.715977  1076 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "59cc37f7ef9c409493bd89d0d2097339" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:01.718392  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "600a18b7e5104a78935d906b8adfde6f" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:01:01.737668   497 catalog_manager.cc:6146] Not sending DeleteTablet requests; no consensus state for tablet df90c4151aa44b83a12b17680dc161da
I20261120 03:01:01.738009   497 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:45640 (request call id 176) took 1161 ms. Trace:
I20261120 03:01:01.745354   497 rpcz_store.cc:276] 1120 03:01:00.576183 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:00.576286 (+   103us) service_pool.cc:224] Handling call
1120 03:01:00.577485 (+  1199us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:01:00.577979 (+   494us) catalog_manager.cc:2859] Modifying in-memory table state
1120 03:01:00.578116 (+   137us) catalog_manager.cc:2868] Locking tablets
1120 03:01:00.578571 (+   455us) catalog_manager.cc:2882] Removing table and tablets from system table
1120 03:01:00.992698 (+414127us) catalog_manager.cc:2900] Removing table from by-name map
1120 03:01:00.992870 (+   172us) catalog_manager.cc:2916] Committing in-memory state
1120 03:01:01.737559 (+744689us) catalog_manager.cc:2920] Aborting table tasks
1120 03:01:01.737955 (+   396us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:00.905782 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:00.907043 (+  1261us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:00.907071 (+    28us) write_op.cc:440] Acquired schema lock
1120 03:01:00.967270 (+ 60199us) tablet.cc:710] Decoding operations
1120 03:01:00.967628 (+   358us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:00.967668 (+    40us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:00.967701 (+    33us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:01:00.967908 (+   207us) tablet.cc:758] Row locks acquired
1120 03:01:00.967921 (+    13us) write_op.cc:260] PREPARE: finished
1120 03:01:00.968038 (+   117us) write_op.cc:270] Start()
1120 03:01:00.968118 (+    80us) write_op.cc:276] Timestamp: P: 1795143660967978 usec, L: 0
1120 03:01:00.968134 (+    16us) op_driver.cc:350] REPLICATION: starting
1120 03:01:00.968616 (+   482us) log.cc:844] Serialized 533 byte log entry
1120 03:01:00.977138 (+  8522us) op_driver.cc:466] REPLICATION: finished
1120 03:01:00.981829 (+  4691us) write_op.cc:301] APPLY: starting
1120 03:01:00.981884 (+    55us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:00.981990 (+   106us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:00.982092 (+   102us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:00.982313 (+   221us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:00.989316 (+  7003us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
1120 03:01:00.989362 (+    46us) write_op.cc:312] APPLY: finished
1120 03:01:00.989557 (+   195us) log.cc:844] Serialized 29 byte log entry
1120 03:01:00.991447 (+  1890us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:00.991517 (+    70us) write_op.cc:459] Released schema lock
1120 03:01:00.991548 (+    31us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:01.011590 (+ 20042us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:01.011614 (+    24us) write_op.cc:459] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":4534,"apply.run_cpu_time_us":1389,"apply.run_wall_time_us":22754,"num_ops":2,"prepare.queue_time_us":325509,"prepare.run_cpu_time_us":2265,"prepare.run_wall_time_us":105843,"raft.queue_time_us":90,"raft.run_cpu_time_us":801,"raft.run_wall_time_us":6708,"replication_time_us":8903}]]}
I20261120 03:01:01.749840   497 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_5e1840985084419290ec475c0971cfe4"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:01:01.751562   497 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_5e1840985084419290ec475c0971cfe4 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:01:01.789563   857 leader_election.cc:343] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:01.728734  1181 raft_consensus.cc:366] T a04da9cb8f004443aa380902ea5427c5 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:01.790462  1181 raft_consensus.cc:392] T a04da9cb8f004443aa380902ea5427c5 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
W20261120 03:01:01.791119   857 leader_election.cc:343] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:01.790860  1181 raft_consensus.cc:787] T a04da9cb8f004443aa380902ea5427c5 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
W20261120 03:01:01.791668   857 leader_election.cc:343] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:01:01.792161   857 leader_election.cc:343] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:01.792371  1181 consensus_queue.cc:261] T a04da9cb8f004443aa380902ea5427c5 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:01.737978  1837 raft_consensus.cc:556] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:01.803076  1875 tablet.cc:1872] T d736d65c3e0b419084b4934dad83f16a P 5beeefe8bef54caeac463d05d8c30fa6: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20261120 03:01:01.803715  1837 leader_election.cc:290] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:01.792992  1747 raft_consensus.cc:534] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:01.804422  1747 raft_consensus.cc:556] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:01.734463   710 leader_election.cc:304] T 5da3d2724dd34aed88e6b08983188004 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:01.806751  1747 leader_election.cc:290] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
W20261120 03:01:01.807559   709 leader_election.cc:343] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:01.809437  1180 tablet_bootstrap.cc:492] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:01:01.809897  1180 ts_tablet_manager.cc:1449] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.642s	user 0.014s	sys 0.000s
W20261120 03:01:01.822935   709 leader_election.cc:343] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:01.824544   502 catalog_manager.cc:5856] T f376fdc03b3b4e3680171af6b8574c2e P 3d30c2428730481db9b0ffd279dd3440 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66). New cstate: current_term: 1 leader_uuid: "3d30c2428730481db9b0ffd279dd3440" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: HEALTHY } } }
I20261120 03:01:01.826865   502 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 1faba22e8e38418383b86e42e904a761 (table default.Table_47101d55d8f6485c816a02a54fda67a8 [id=252d229ff1334cd99ec99e5c921031bb]). Expected version 1 got 0
I20261120 03:01:01.827747   502 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
I20261120 03:01:01.833791  1076 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b549a06d15db449b851fe9549a963416" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:01.829551  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9dadc05b726346309d6b350f9abb4abb" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:01:01.836760  1073 raft_consensus.cc:2062] T a04da9cb8f004443aa380902ea5427c5 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:01:01.813997  1180 raft_consensus.cc:366] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:01.838749  1180 raft_consensus.cc:392] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:01.839040  1180 raft_consensus.cc:787] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:01.839663  1180 consensus_queue.cc:261] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
W20261120 03:01:01.842325   857 leader_election.cc:343] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:01:01.843770   857 leader_election.cc:343] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:01.846131  1677 ts_manager.cc:210] Registered new tserver with Master: 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:01.850777  1761 ts_tablet_manager.cc:1996] T 020be7f7cda0467aab56f657cbe8d0ad P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:01:01.855815  1806 raft_consensus.cc:2966] T 5da3d2724dd34aed88e6b08983188004 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20261120 03:01:01.856761   710 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 122853us
I20261120 03:01:01.857059   502 catalog_manager.cc:5856] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440 reported cstate change: term changed from 0 to 1, leader changed from <none> to 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67). New cstate: current_term: 1 leader_uuid: "823abdbd90ba47bbacad0f8dcff78234" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } }
I20261120 03:01:01.858389  1761 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:52644 (request call id 135) took 1979 ms. Trace:
I20261120 03:01:01.858428   502 catalog_manager.cc:5856] T 091dd73d27df4db09ab750747da7584e P 3d30c2428730481db9b0ffd279dd3440 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66). New cstate: current_term: 1 leader_uuid: "3d30c2428730481db9b0ffd279dd3440" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: HEALTHY } } }
I20261120 03:01:01.858721  1761 rpcz_store.cc:276] 1120 03:00:59.879435 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.879843 (+   408us) service_pool.cc:224] Handling call
1120 03:01:00.768869 (+889026us) ts_tablet_manager.cc:1197] Deleting tablet 020be7f7cda0467aab56f657cbe8d0ad
1120 03:01:00.789030 (+ 20161us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:01.265294 (+476264us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:01.682777 (+417483us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:01.858324 (+175547us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":4132,"tablet-delete.queue_time_us":888409}
I20261120 03:01:01.860042  1761 tablet_replica.cc:333] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:01:01.860196   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 020be7f7cda0467aab56f657cbe8d0ad (table default.Table_d1737969bdd644a6ab39da121eeedd60 [id=ed325a58ff124ba9a6b1d05bc901793b]) successfully deleted
I20261120 03:01:01.860832  1761 raft_consensus.cc:2440] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:01:01.866711  1181 ts_tablet_manager.cc:1480] T a04da9cb8f004443aa380902ea5427c5 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.186s	user 0.008s	sys 0.001s
I20261120 03:01:01.873561   710 leader_election.cc:304] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:01.877028  1324 raft_consensus.cc:366] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:01.893165  1324 raft_consensus.cc:392] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:01.893461  1324 raft_consensus.cc:787] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:01:01.894203  1324 consensus_queue.cc:261] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:01.896471  1324 ts_tablet_manager.cc:1480] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.166s	user 0.005s	sys 0.000s
I20261120 03:01:01.897370  1324 tablet_bootstrap.cc:492] T 29e6a50c9e1f4b778737abdef9ad073a P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:01:01.891911  1761 raft_consensus.cc:2469] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:01.903499  1180 ts_tablet_manager.cc:1480] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.092s	user 0.005s	sys 0.000s
I20261120 03:01:01.904767  1761 ts_tablet_manager.cc:1962] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:01.906459  1806 raft_consensus.cc:3024] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:01.906807  1806 raft_consensus.cc:534] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:01.907138  1806 raft_consensus.cc:3296] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:01.907948  1806 raft_consensus.cc:556] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:01.909263   710 leader_election.cc:304] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:01.911304  1808 raft_consensus.cc:3024] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:01.911659  1808 raft_consensus.cc:534] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:01.911960  1808 raft_consensus.cc:3296] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:01.913560  1181 tablet_bootstrap.cc:492] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:01.926204  1180 tablet_bootstrap.cc:492] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:01:01.932482  1808 raft_consensus.cc:556] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:01.934324   857 leader_election.cc:304] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:01.935616  1791 raft_consensus.cc:3296] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:01.935724   857 leader_election.cc:304] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:01.936429  1791 raft_consensus.cc:2966] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:01.939174  1850 tablet.cc:1872] T d736d65c3e0b419084b4934dad83f16a P 3d30c2428730481db9b0ffd279dd3440: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20261120 03:01:01.939860   774 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a04da9cb8f004443aa380902ea5427c5" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:01.940572   774 raft_consensus.cc:2628] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 5beeefe8bef54caeac463d05d8c30fa6 in this term.
I20261120 03:01:01.941401   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "600a18b7e5104a78935d906b8adfde6f" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:01.942446   773 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "59cc37f7ef9c409493bd89d0d2097339" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:01.943089   773 raft_consensus.cc:2062] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:01:01.947211  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "600a18b7e5104a78935d906b8adfde6f" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:01:01.942224  1806 leader_election.cc:290] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:01:01.948290  1324 tablet_bootstrap.cc:654] T 29e6a50c9e1f4b778737abdef9ad073a P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:01.949656   777 raft_consensus.cc:2628] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 5beeefe8bef54caeac463d05d8c30fa6 in this term.
W20261120 03:01:01.950709   709 leader_election.cc:343] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:01.954391  1747 raft_consensus.cc:3024] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:01.954623   772 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9dadc05b726346309d6b350f9abb4abb" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:01.954885  1747 raft_consensus.cc:744] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:01:01.955199  1180 tablet_bootstrap.cc:654] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:01.955206   772 raft_consensus.cc:3296] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:01.955648  1747 consensus_queue.cc:238] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:01.956233   772 raft_consensus.cc:2062] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:01:01.961448   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "600a18b7e5104a78935d906b8adfde6f" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:01:01.962761   920 raft_consensus.cc:2628] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 5beeefe8bef54caeac463d05d8c30fa6 in current term 1: Already voted for candidate 3d30c2428730481db9b0ffd279dd3440 in this term.
I20261120 03:01:01.966862  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a04da9cb8f004443aa380902ea5427c5" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:01:01.967411  1075 raft_consensus.cc:3296] T a04da9cb8f004443aa380902ea5427c5 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:01.968356  1075 raft_consensus.cc:2062] T a04da9cb8f004443aa380902ea5427c5 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 1
I20261120 03:01:01.972204  1180 tablet_bootstrap.cc:492] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:01:01.972325  1181 tablet_bootstrap.cc:654] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:01.972642  1180 ts_tablet_manager.cc:1449] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.047s	user 0.008s	sys 0.001s
I20261120 03:01:01.976728   857 leader_election.cc:304] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:01.977573  1796 raft_consensus.cc:2966] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20261120 03:01:01.977932  1808 leader_election.cc:290] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:01:01.978560   857 leader_election.cc:304] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:01.979466  1796 raft_consensus.cc:3024] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:01.980293  1796 raft_consensus.cc:534] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:01.980603  1796 raft_consensus.cc:3296] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:01.981321  1796 raft_consensus.cc:556] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:01.981582   710 leader_election.cc:304] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:01.986625  1886 raft_consensus.cc:2966] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20261120 03:01:01.986726  1180 raft_consensus.cc:366] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:01.987330  1180 raft_consensus.cc:392] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:01.987597  1180 raft_consensus.cc:787] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:01.990551  1796 leader_election.cc:290] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:01.991572   857 leader_election.cc:304] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
W20261120 03:01:01.992731   857 leader_election.cc:343] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: b549a06d15db449b851fe9549a963416
I20261120 03:01:01.993083   857 leader_election.cc:304] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:01.993734   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "a04da9cb8f004443aa380902ea5427c5" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:01:01.994494   920 raft_consensus.cc:2628] T a04da9cb8f004443aa380902ea5427c5 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 5beeefe8bef54caeac463d05d8c30fa6 in current term 1: Already voted for candidate 3d30c2428730481db9b0ffd279dd3440 in this term.
I20261120 03:01:01.996161   709 leader_election.cc:304] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:01:01.996982  1886 raft_consensus.cc:3024] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:01.996833  1180 consensus_queue.cc:261] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:01.997586  1886 raft_consensus.cc:744] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Becoming Leader. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Running, Role: LEADER
I20261120 03:01:01.998391  1876 raft_consensus.cc:2966] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20261120 03:01:01.999361  1876 raft_consensus.cc:2966] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:01.999280  1886 consensus_queue.cc:238] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:02.000612  1181 tablet_bootstrap.cc:492] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:02.008067  1180 ts_tablet_manager.cc:1480] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.035s	user 0.006s	sys 0.001s
I20261120 03:01:02.008421  1324 tablet_bootstrap.cc:492] T 29e6a50c9e1f4b778737abdef9ad073a P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:01:02.008800  1324 ts_tablet_manager.cc:1449] T 29e6a50c9e1f4b778737abdef9ad073a P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.112s	user 0.010s	sys 0.003s
I20261120 03:01:02.008935  1180 tablet_bootstrap.cc:492] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:01:02.012976  1324 raft_consensus.cc:366] T 29e6a50c9e1f4b778737abdef9ad073a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.013593  1324 raft_consensus.cc:392] T 29e6a50c9e1f4b778737abdef9ad073a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:02.013823  1324 raft_consensus.cc:787] T 29e6a50c9e1f4b778737abdef9ad073a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:01:02.015358  1181 ts_tablet_manager.cc:1449] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.102s	user 0.009s	sys 0.003s
I20261120 03:01:02.016312   772 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "59cc37f7ef9c409493bd89d0d2097339" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:02.016834   772 raft_consensus.cc:3296] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:02.017812   772 raft_consensus.cc:2062] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:01:02.022331  1180 tablet_bootstrap.cc:654] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:02.024231  1324 consensus_queue.cc:261] T 29e6a50c9e1f4b778737abdef9ad073a P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.026861  1181 raft_consensus.cc:366] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:02.027465  1181 raft_consensus.cc:392] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:02.027688  1181 raft_consensus.cc:787] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:02.028264  1181 consensus_queue.cc:261] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:02.035276  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "59cc37f7ef9c409493bd89d0d2097339" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:01:02.041172  1761 ts_tablet_manager.cc:1975] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:02.041550  1761 log.cc:1199] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/5c5c0ac869a1419fad9f61ec647027ee
I20261120 03:01:02.048076   857 leader_election.cc:304] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:02.050734  1324 ts_tablet_manager.cc:1480] T 29e6a50c9e1f4b778737abdef9ad073a P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.042s	user 0.008s	sys 0.000s
I20261120 03:01:02.051658  1324 tablet_bootstrap.cc:492] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:01:02.054299  1853 raft_consensus.cc:3024] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:02.055688  1853 raft_consensus.cc:744] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:01:02.056444  1853 consensus_queue.cc:238] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:02.066958  1181 ts_tablet_manager.cc:1480] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.051s	user 0.005s	sys 0.000s
I20261120 03:01:02.067829  1181 tablet_bootstrap.cc:492] T 091dd73d27df4db09ab750747da7584e P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:02.073657  1761 ts_tablet_manager.cc:1996] T 5c5c0ac869a1419fad9f61ec647027ee P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
W20261120 03:01:02.077457   857 leader_election.cc:343] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:01:02.078118   857 consensus_peers.cc:597] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:01:02.081434  1761 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:52644 (request call id 138) took 2201 ms. Trace:
I20261120 03:01:02.084652  1324 tablet_bootstrap.cc:654] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:02.093600  1180 tablet_bootstrap.cc:492] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:01:02.094127  1180 ts_tablet_manager.cc:1449] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.085s	user 0.008s	sys 0.006s
I20261120 03:01:02.081758  1761 rpcz_store.cc:276] 1120 03:00:59.880355 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.880579 (+   224us) service_pool.cc:224] Handling call
1120 03:01:01.859883 (+1979304us) ts_tablet_manager.cc:1197] Deleting tablet 5c5c0ac869a1419fad9f61ec647027ee
1120 03:01:01.859938 (+    55us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:01.984123 (+124185us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:02.041038 (+ 56915us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:02.081376 (+ 40338us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1978711}
I20261120 03:01:02.095355   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 5c5c0ac869a1419fad9f61ec647027ee (table default.Table_5c5911c8746d4b26843ec77c8141d3b6 [id=ffb2040a2e114893855d4d8c66dfa855]) successfully deleted
I20261120 03:01:02.097092  1761 tablet_replica.cc:333] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:01:02.118292  1181 tablet_bootstrap.cc:654] T 091dd73d27df4db09ab750747da7584e P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:02.128597  1181 tablet_bootstrap.cc:492] T 091dd73d27df4db09ab750747da7584e P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:02.129243  1181 ts_tablet_manager.cc:1449] T 091dd73d27df4db09ab750747da7584e P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.062s	user 0.005s	sys 0.004s
I20261120 03:01:02.132738  1761 raft_consensus.cc:2440] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:01:02.133780  1761 raft_consensus.cc:2469] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:02.137885  1847 catalog_manager.cc:1310] Loaded cluster ID: 96a176325a9f4b23b7e54922a0ed8e6d
I20261120 03:01:02.141511  1806 consensus_queue.cc:1049] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:02.109773  1180 raft_consensus.cc:366] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.143412  1180 raft_consensus.cc:392] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:02.143671  1180 raft_consensus.cc:787] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:02.144282  1809 raft_consensus.cc:534] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:02.144255  1180 consensus_queue.cc:261] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.144687  1809 raft_consensus.cc:556] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.145049  1853 consensus_queue.cc:1049] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:02.146055  1847 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f: loading cluster ID for follower catalog manager: success
I20261120 03:01:02.146552  1809 leader_election.cc:290] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:02.146538   857 leader_election.cc:304] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:02.146960  1809 consensus_queue.cc:1049] T a04da9cb8f004443aa380902ea5427c5 P 5beeefe8bef54caeac463d05d8c30fa6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:02.149253  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ad49c8e00aca4a88a091baceae320477" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:01:02.150581   709 leader_election.cc:343] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:02.154258  1853 raft_consensus.cc:3024] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:02.154683  1853 raft_consensus.cc:744] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:01:02.155488  1853 consensus_queue.cc:238] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.157006  1761 ts_tablet_manager.cc:1962] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:02.160552  1837 consensus_queue.cc:1049] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:02.160921   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ad49c8e00aca4a88a091baceae320477" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:02.161522  1324 tablet_bootstrap.cc:492] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:01:02.161955  1324 ts_tablet_manager.cc:1449] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.110s	user 0.005s	sys 0.006s
W20261120 03:01:02.162314   710 leader_election.cc:343] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:02.162688   710 leader_election.cc:304] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:02.164647  1809 raft_consensus.cc:2966] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:02.165534  1809 raft_consensus.cc:534] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:02.165730  1180 ts_tablet_manager.cc:1480] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.071s	user 0.004s	sys 0.000s
I20261120 03:01:02.166005  1809 raft_consensus.cc:556] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.168794   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00f7c7d95bae4d55be6b437701989ba3" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:02.169739  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00f7c7d95bae4d55be6b437701989ba3" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:01:02.170974   709 leader_election.cc:343] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:02.168716  1809 leader_election.cc:290] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:02.179332  1324 raft_consensus.cc:366] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.184005   497 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
W20261120 03:01:02.203120   857 consensus_peers.cc:597] T 9dadc05b726346309d6b350f9abb4abb P 3d30c2428730481db9b0ffd279dd3440 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:01:02.204635  1180 tablet_bootstrap.cc:492] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:01:02.206326  1181 raft_consensus.cc:366] T 091dd73d27df4db09ab750747da7584e P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:02.206933  1181 raft_consensus.cc:392] T 091dd73d27df4db09ab750747da7584e P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:02.179941  1324 raft_consensus.cc:392] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:02.211570  1324 raft_consensus.cc:787] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
W20261120 03:01:02.181456   710 leader_election.cc:343] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:02.212505   710 leader_election.cc:304] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:02.212311  1324 consensus_queue.cc:261] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.213184  1181 raft_consensus.cc:787] T 091dd73d27df4db09ab750747da7584e P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:02.213851  1808 raft_consensus.cc:2966] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:02.231767  1324 ts_tablet_manager.cc:1480] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.069s	user 0.004s	sys 0.000s
I20261120 03:01:02.232364  1181 consensus_queue.cc:261] T 091dd73d27df4db09ab750747da7584e P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:02.237142  1181 ts_tablet_manager.cc:1480] T 091dd73d27df4db09ab750747da7584e P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.108s	user 0.005s	sys 0.000s
W20261120 03:01:02.239168   857 consensus_peers.cc:597] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:01:02.246044  1324 tablet_bootstrap.cc:492] T 4da5eb2f2f114cc792488c5c74bb1414 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:01:02.250932  1853 consensus_queue.cc:1049] T 59cc37f7ef9c409493bd89d0d2097339 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20261120 03:01:02.261690  1045 tablet_service.cc:1495] Processing CreateTablet for tablet 3790f81eac6943ef9764a9b9c11ed7f7 (DEFAULT_TABLE table=default.Table_902789719933401f96796eebfca3fc97 [id=afed15c957ff48dda833fe91fe366399]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.265537  1181 tablet_bootstrap.cc:492] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:02.265851  1045 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3790f81eac6943ef9764a9b9c11ed7f7. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.275707  1847 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f: acquiring CA information for follower catalog manager: success
I20261120 03:01:02.277199  1853 raft_consensus.cc:534] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:02.278766  1853 raft_consensus.cc:556] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.280598  1853 leader_election.cc:290] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:02.281637  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0179a59749a14168a840e9d4e610be1f" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:02.283078  1893 raft_consensus.cc:534] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:02.283476  1893 raft_consensus.cc:556] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:02.284029   898 tablet_service.cc:1495] Processing CreateTablet for tablet 3790f81eac6943ef9764a9b9c11ed7f7 (DEFAULT_TABLE table=default.Table_902789719933401f96796eebfca3fc97 [id=afed15c957ff48dda833fe91fe366399]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.284996   898 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3790f81eac6943ef9764a9b9c11ed7f7. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.285364  1893 leader_election.cc:290] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:01:02.284943   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0179a59749a14168a840e9d4e610be1f" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:02.286002   777 raft_consensus.cc:2062] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
W20261120 03:01:02.286535  1011 leader_election.cc:343] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: b549a06d15db449b851fe9549a963416
I20261120 03:01:02.286782   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b549a06d15db449b851fe9549a963416" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:02.287412   919 raft_consensus.cc:2062] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:01:02.288669  1012 leader_election.cc:304] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:01:02.289397  1180 tablet_bootstrap.cc:654] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:02.289738  1893 raft_consensus.cc:3024] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:02.290032  1893 raft_consensus.cc:534] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:02.290346  1893 raft_consensus.cc:3296] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:02.291036  1893 raft_consensus.cc:556] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:02.295302  1181 tablet_bootstrap.cc:654] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:02.297449  1324 tablet_bootstrap.cc:654] T 4da5eb2f2f114cc792488c5c74bb1414 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:02.299535   857 leader_election.cc:304] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:01:02.303097  1893 leader_election.cc:290] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:01:02.303395  1876 raft_consensus.cc:3024] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:02.303714  1876 raft_consensus.cc:534] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:02.303633   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "b549a06d15db449b851fe9549a963416" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:01:02.304097  1876 raft_consensus.cc:3296] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:02.304263   919 raft_consensus.cc:3296] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:02.305235   919 raft_consensus.cc:2062] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
W20261120 03:01:02.305740  1011 leader_election.cc:343] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: b549a06d15db449b851fe9549a963416
I20261120 03:01:02.304893  1876 raft_consensus.cc:556] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.313834  1012 leader_election.cc:304] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:01:02.314227  1876 leader_election.cc:290] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:02.315127  1893 raft_consensus.cc:3024] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:02.315680  1893 raft_consensus.cc:744] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Becoming Leader. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Running, Role: LEADER
I20261120 03:01:02.319849  1761 ts_tablet_manager.cc:1975] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:02.321702  1761 log.cc:1199] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/a08aa5ddcf1a4061a70275a8eecb1856
I20261120 03:01:02.322733  1893 consensus_queue.cc:238] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:02.329483   501 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
I20261120 03:01:02.325273   753 tablet_service.cc:1495] Processing CreateTablet for tablet 3790f81eac6943ef9764a9b9c11ed7f7 (DEFAULT_TABLE table=default.Table_902789719933401f96796eebfca3fc97 [id=afed15c957ff48dda833fe91fe366399]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.331981   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 3790f81eac6943ef9764a9b9c11ed7f7. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.332324  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0179a59749a14168a840e9d4e610be1f" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:01:02.336308   772 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0179a59749a14168a840e9d4e610be1f" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:02.336887   772 raft_consensus.cc:3296] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:02.338769   501 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 1faba22e8e38418383b86e42e904a761 (table default.Table_47101d55d8f6485c816a02a54fda67a8 [id=252d229ff1334cd99ec99e5c921031bb]). Expected version 1 got 0
I20261120 03:01:02.340283   501 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet d736d65c3e0b419084b4934dad83f16a (table default.Table_04ddab7f706b43e7ba12fa247b3ca933 [id=9f571a69327a498e89f323c9ac0eee34]). Expected version 1 got 0
I20261120 03:01:02.326563  1809 raft_consensus.cc:534] T 29e6a50c9e1f4b778737abdef9ad073a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:02.342435  1809 raft_consensus.cc:556] T 29e6a50c9e1f4b778737abdef9ad073a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.345175   772 raft_consensus.cc:2062] T 0179a59749a14168a840e9d4e610be1f P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:01:02.345927   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "29e6a50c9e1f4b778737abdef9ad073a" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:02.346550  1051 tablet_service.cc:1495] Processing CreateTablet for tablet 9a17bd00b1eb492f8e576f4e8b563c03 (DEFAULT_TABLE table=default.Table_a887fc0441b4418eac5a6bca56a83110 [id=c57c0076f53c4af889a8d2a53739aab1]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.347661  1051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9a17bd00b1eb492f8e576f4e8b563c03. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:01:02.348584   710 leader_election.cc:343] T 29e6a50c9e1f4b778737abdef9ad073a P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:02.348898   895 tablet_service.cc:1495] Processing CreateTablet for tablet 9a17bd00b1eb492f8e576f4e8b563c03 (DEFAULT_TABLE table=default.Table_a887fc0441b4418eac5a6bca56a83110 [id=c57c0076f53c4af889a8d2a53739aab1]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.349963   895 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9a17bd00b1eb492f8e576f4e8b563c03. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.350448  1809 leader_election.cc:290] T 29e6a50c9e1f4b778737abdef9ad073a P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:02.350646  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "29e6a50c9e1f4b778737abdef9ad073a" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:01:02.351974   709 leader_election.cc:343] T 29e6a50c9e1f4b778737abdef9ad073a P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:02.352316   709 leader_election.cc:304] T 29e6a50c9e1f4b778737abdef9ad073a P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:02.353160  1886 raft_consensus.cc:2966] T 29e6a50c9e1f4b778737abdef9ad073a P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:02.359122  1761 ts_tablet_manager.cc:1996] T a08aa5ddcf1a4061a70275a8eecb1856 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:01:02.364744  1853 raft_consensus.cc:534] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
W20261120 03:01:02.361872   857 leader_election.cc:343] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:01:02.365382  1011 consensus_peers.cc:597] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 -> Peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Couldn't send request to peer 5beeefe8bef54caeac463d05d8c30fa6. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b549a06d15db449b851fe9549a963416. This is attempt 1: this message will repeat every 5th retry.
W20261120 03:01:02.366145   857 leader_election.cc:343] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:02.366220  1853 raft_consensus.cc:556] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.368110  1853 leader_election.cc:290] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:02.370832  1893 consensus_queue.cc:1049] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:02.375401  1181 tablet_bootstrap.cc:492] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:02.375814  1181 ts_tablet_manager.cc:1449] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.110s	user 0.006s	sys 0.003s
I20261120 03:01:02.376384  1886 raft_consensus.cc:534] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:02.376914  1886 raft_consensus.cc:556] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.379436   857 leader_election.cc:304] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:02.381089  1876 raft_consensus.cc:3024] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:02.381500  1876 raft_consensus.cc:744] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:01:02.382251  1876 consensus_queue.cc:238] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.384239   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3bc0f2f9cb7945ab9b0ead61c934f9dd" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:02.385004   920 raft_consensus.cc:2062] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 5beeefe8bef54caeac463d05d8c30fa6 in term 0
I20261120 03:01:02.386430  1180 tablet_bootstrap.cc:492] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:01:02.386816  1180 ts_tablet_manager.cc:1449] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.182s	user 0.009s	sys 0.001s
I20261120 03:01:02.387480  1886 leader_election.cc:290] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:02.388885  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3bc0f2f9cb7945ab9b0ead61c934f9dd" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:02.388185   756 tablet_service.cc:1495] Processing CreateTablet for tablet 9a17bd00b1eb492f8e576f4e8b563c03 (DEFAULT_TABLE table=default.Table_a887fc0441b4418eac5a6bca56a83110 [id=c57c0076f53c4af889a8d2a53739aab1]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.392546   902 tablet_service.cc:1495] Processing CreateTablet for tablet aceaf951618a4aabb2b858e4cf6a4f88 (DEFAULT_TABLE table=default.Table_5415c198c7814e0f988683d59c9181d3 [id=3e126deb5b124b8ab9244c7ade109408]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.393411   756 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9a17bd00b1eb492f8e576f4e8b563c03. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.393617   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet aceaf951618a4aabb2b858e4cf6a4f88. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.389092  1180 raft_consensus.cc:366] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.393884  1180 raft_consensus.cc:392] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:02.394153  1180 raft_consensus.cc:787] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:02.394703  1180 consensus_queue.cc:261] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.397203  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3bc0f2f9cb7945ab9b0ead61c934f9dd" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:02.391968   710 leader_election.cc:304] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
W20261120 03:01:02.400506   857 leader_election.cc:343] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:01:02.400903   709 leader_election.cc:343] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:02.402595  1886 raft_consensus.cc:3024] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:02.403801   772 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3bc0f2f9cb7945ab9b0ead61c934f9dd" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:02.405292   772 raft_consensus.cc:2062] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:01:02.405900  1886 raft_consensus.cc:534] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:02.406641  1180 ts_tablet_manager.cc:1480] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.020s	user 0.006s	sys 0.000s
I20261120 03:01:02.407508  1180 tablet_bootstrap.cc:492] T ad49c8e00aca4a88a091baceae320477 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:01:02.414002  1051 tablet_service.cc:1495] Processing CreateTablet for tablet aceaf951618a4aabb2b858e4cf6a4f88 (DEFAULT_TABLE table=default.Table_5415c198c7814e0f988683d59c9181d3 [id=3e126deb5b124b8ab9244c7ade109408]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.415082  1051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet aceaf951618a4aabb2b858e4cf6a4f88. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.417083  1886 raft_consensus.cc:3296] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:02.421337   501 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 0179a59749a14168a840e9d4e610be1f (table default.Table_cc56bf0ad8344f1a94d5da6e4d77e8c0 [id=b9034a083c7e4133a746b336335d3e42]). Expected version 1 got 0
I20261120 03:01:02.418421  1886 raft_consensus.cc:556] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
W20261120 03:01:02.426631   495 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 73c993944b0a4977b55d40ad63bb69e3 consensus state has no leader
I20261120 03:01:02.427150   495 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:45640 (request call id 178) took 1635 ms. Trace:
I20261120 03:01:02.427354  1324 tablet_bootstrap.cc:492] T 4da5eb2f2f114cc792488c5c74bb1414 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:01:02.427747  1324 ts_tablet_manager.cc:1449] T 4da5eb2f2f114cc792488c5c74bb1414 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.182s	user 0.003s	sys 0.008s
I20261120 03:01:02.427403   495 rpcz_store.cc:276] 1120 03:01:00.791667 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:00.918629 (+126962us) service_pool.cc:224] Handling call
1120 03:01:00.919449 (+   820us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:01:00.920106 (+   657us) catalog_manager.cc:3769] Apply alter schema
1120 03:01:00.920678 (+   572us) catalog_manager.cc:3801] Acquired catalog manager lock
1120 03:01:00.920871 (+   193us) catalog_manager.cc:3972] Updating metadata on disk
1120 03:01:01.131609 (+210738us) catalog_manager.cc:4003] Committing alterations to in-memory state
1120 03:01:02.427095 (+1295486us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:01.011879 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:01.012263 (+   384us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:01.012287 (+    24us) write_op.cc:440] Acquired schema lock
1120 03:01:01.012300 (+    13us) tablet.cc:710] Decoding operations
1120 03:01:01.012555 (+   255us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:01.012593 (+    38us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:01.012615 (+    22us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:01:01.012766 (+   151us) tablet.cc:758] Row locks acquired
1120 03:01:01.012778 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:01:01.012882 (+   104us) write_op.cc:270] Start()
1120 03:01:01.012956 (+    74us) write_op.cc:276] Timestamp: P: 1795143661012822 usec, L: 0
1120 03:01:01.012972 (+    16us) op_driver.cc:350] REPLICATION: starting
1120 03:01:01.013435 (+   463us) log.cc:844] Serialized 405 byte log entry
1120 03:01:01.112669 (+ 99234us) op_driver.cc:466] REPLICATION: finished
1120 03:01:01.121806 (+  9137us) write_op.cc:301] APPLY: starting
1120 03:01:01.121867 (+    61us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:01.121953 (+    86us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:01.122055 (+   102us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:01.122216 (+   161us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:01.122346 (+   130us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:01:01.122390 (+    44us) write_op.cc:312] APPLY: finished
1120 03:01:01.122569 (+   179us) log.cc:844] Serialized 23 byte log entry
1120 03:01:01.131046 (+  8477us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:01.131102 (+    56us) write_op.cc:459] Released schema lock
1120 03:01:01.131132 (+    30us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:01.131924 (+   792us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:01.131947 (+    23us) write_op.cc:459] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":8958,"apply.run_cpu_time_us":1218,"apply.run_wall_time_us":10088,"num_ops":1,"prepare.queue_time_us":89956,"prepare.run_cpu_time_us":1886,"prepare.run_wall_time_us":65194,"raft.queue_time_us":100,"raft.run_cpu_time_us":760,"raft.run_wall_time_us":760,"replication_time_us":99583}]]}
I20261120 03:01:02.430495   857 leader_election.cc:304] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:02.432186  1796 raft_consensus.cc:3024] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:02.432238  1045 tablet_service.cc:1495] Processing CreateTablet for tablet 1d00f5c083c44b47bbf004b1c5dfcc1f (DEFAULT_TABLE table=default.Table_58015d78e3d24cd79830293a3ce6176c [id=49c74781c4bd4acf962012a77f3d3b1b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.432510  1796 raft_consensus.cc:534] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:02.432821  1796 raft_consensus.cc:3296] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:02.432617   750 tablet_service.cc:1495] Processing CreateTablet for tablet aceaf951618a4aabb2b858e4cf6a4f88 (DEFAULT_TABLE table=default.Table_5415c198c7814e0f988683d59c9181d3 [id=3e126deb5b124b8ab9244c7ade109408]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.433248  1045 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1d00f5c083c44b47bbf004b1c5dfcc1f. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.433687   750 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet aceaf951618a4aabb2b858e4cf6a4f88. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.433522  1796 raft_consensus.cc:556] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.442972   495 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_9821147009834f8482ae7e45dfd18e3c"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:01:02.444689   495 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_9821147009834f8482ae7e45dfd18e3c in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:01:02.461655   495 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:45640 (request call id 184) took 1012 ms. Trace:
I20261120 03:01:02.465818  1761 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:52644 (request call id 139) took 2485 ms. Trace:
I20261120 03:01:02.466173   495 rpcz_store.cc:276] 1120 03:01:01.449193 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:02.442610 (+993417us) service_pool.cc:224] Handling call
1120 03:01:02.445208 (+  2598us) catalog_manager.cc:2145] Acquired catalog manager lock
1120 03:01:02.446650 (+  1442us) catalog_manager.cc:2191] Created new table and tablet info
1120 03:01:02.461009 (+ 14359us) catalog_manager.cc:2251] Wrote table and tablets to system table
1120 03:01:02.461321 (+   312us) catalog_manager.cc:2285] Inserted table and tablets into CatalogManager maps
1120 03:01:02.461595 (+   274us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:02.448073 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:02.448474 (+   401us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:02.448499 (+    25us) write_op.cc:440] Acquired schema lock
1120 03:01:02.448513 (+    14us) tablet.cc:710] Decoding operations
1120 03:01:02.448786 (+   273us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:02.448825 (+    39us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:02.448847 (+    22us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:01:02.449055 (+   208us) tablet.cc:758] Row locks acquired
1120 03:01:02.449078 (+    23us) write_op.cc:260] PREPARE: finished
1120 03:01:02.449193 (+   115us) write_op.cc:270] Start()
1120 03:01:02.449274 (+    81us) write_op.cc:276] Timestamp: P: 1795143662449139 usec, L: 0
1120 03:01:02.449291 (+    17us) op_driver.cc:350] REPLICATION: starting
1120 03:01:02.449766 (+   475us) log.cc:844] Serialized 433 byte log entry
1120 03:01:02.459168 (+  9402us) op_driver.cc:466] REPLICATION: finished
1120 03:01:02.459490 (+   322us) write_op.cc:301] APPLY: starting
1120 03:01:02.459543 (+    53us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:02.459655 (+   112us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:02.459762 (+   107us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:02.460174 (+   412us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:02.460309 (+   135us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
1120 03:01:02.460350 (+    41us) write_op.cc:312] APPLY: finished
1120 03:01:02.460518 (+   168us) log.cc:844] Serialized 29 byte log entry
1120 03:01:02.460647 (+   129us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:02.460710 (+    63us) write_op.cc:459] Released schema lock
1120 03:01:02.460743 (+    33us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:02.460951 (+   208us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:02.460974 (+    23us) write_op.cc:459] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":180,"apply.run_cpu_time_us":1431,"apply.run_wall_time_us":1429,"num_ops":2,"prepare.queue_time_us":94,"prepare.run_cpu_time_us":2016,"prepare.run_wall_time_us":2017,"raft.queue_time_us":3575,"raft.run_cpu_time_us":1504,"raft.run_wall_time_us":1503,"replication_time_us":9781}]]}
I20261120 03:01:02.470305  1761 rpcz_store.cc:276] 1120 03:00:59.880734 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.881226 (+   492us) service_pool.cc:224] Handling call
1120 03:01:02.096908 (+2215682us) ts_tablet_manager.cc:1197] Deleting tablet a08aa5ddcf1a4061a70275a8eecb1856
1120 03:01:02.096967 (+    59us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:02.251537 (+154570us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:02.319709 (+ 68172us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:02.366024 (+ 46315us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":34834,"tablet-delete.queue_time_us":2215217}
I20261120 03:01:02.470757  1893 raft_consensus.cc:534] T 091dd73d27df4db09ab750747da7584e P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:02.474452   898 tablet_service.cc:1495] Processing CreateTablet for tablet 1d00f5c083c44b47bbf004b1c5dfcc1f (DEFAULT_TABLE table=default.Table_58015d78e3d24cd79830293a3ce6176c [id=49c74781c4bd4acf962012a77f3d3b1b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.475569   898 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1d00f5c083c44b47bbf004b1c5dfcc1f. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.478228  1893 raft_consensus.cc:556] T 091dd73d27df4db09ab750747da7584e P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:02.490218  1893 leader_election.cc:290] T 091dd73d27df4db09ab750747da7584e P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:01:02.493736  1055 tablet_service.cc:1542] Processing DeleteTablet for tablet a6ae8007455d4be3b50672b9c6b0fd8f with delete_type TABLET_DATA_DELETED (replaced by tablet 8f1fda857e6c400e80d3d177a3d748ff) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:02.496074  1046 tablet_service.cc:1542] Processing DeleteTablet for tablet ce612f9a121b4b18908724d2726e156f with delete_type TABLET_DATA_DELETED (replaced by tablet 5da3d2724dd34aed88e6b08983188004) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:02.497576  1046 tablet_service.cc:1542] Processing DeleteTablet for tablet 4abb67f3819a45b9b05dde04842e2c9d with delete_type TABLET_DATA_DELETED (replaced by tablet df90c4151aa44b83a12b17680dc161da) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:02.498992  1046 tablet_service.cc:1542] Processing DeleteTablet for tablet 020be7f7cda0467aab56f657cbe8d0ad with delete_type TABLET_DATA_DELETED (replaced by tablet 600a18b7e5104a78935d906b8adfde6f) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:02.499118  1181 raft_consensus.cc:366] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:02.499740  1181 raft_consensus.cc:392] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:02.499982  1181 raft_consensus.cc:787] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:02.468978   497 catalog_manager.cc:6409] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: default.Table_04ddab7f706b43e7ba12fa247b3ca933 [id=9f571a69327a498e89f323c9ac0eee34] alter complete (version 1)
I20261120 03:01:02.502374  1046 tablet_service.cc:1542] Processing DeleteTablet for tablet a1192a7f050a4f5387a30b2216dec5f0 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:01:00 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:02.503564  1046 tablet_service.cc:1542] Processing DeleteTablet for tablet 200b234da897419a82bf5844620adf24 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:52 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:02.504777  1046 tablet_service.cc:1542] Processing DeleteTablet for tablet 1885397e42774a89bb347bc776284bc0 with delete_type TABLET_DATA_DELETED (replaced by tablet b549a06d15db449b851fe9549a963416) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:02.469542   499 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet ad4e9e0de4954904842bfd5f15f530f0
I20261120 03:01:02.506673  1059 tablet_service.cc:1495] Processing CreateTablet for tablet f8f2fc38086d4e47b1664a23c0cfee21 (DEFAULT_TABLE table=default.Table_184b1c05e67e47718e9f58e229c4c748 [id=e35d83b51ea745ab96033865032acf86]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.507548   499 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:45640 (request call id 168) took 3275 ms. Trace:
I20261120 03:01:02.507757  1059 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f8f2fc38086d4e47b1664a23c0cfee21. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.507851   499 rpcz_store.cc:276] 1120 03:00:59.232344 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.915226 (+682882us) service_pool.cc:224] Handling call
1120 03:01:00.179683 (+264457us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:01:00.180197 (+   514us) catalog_manager.cc:2859] Modifying in-memory table state
1120 03:01:00.180320 (+   123us) catalog_manager.cc:2868] Locking tablets
1120 03:01:00.311952 (+131632us) catalog_manager.cc:2882] Removing table and tablets from system table
1120 03:01:00.555010 (+243058us) catalog_manager.cc:2900] Removing table from by-name map
1120 03:01:00.555227 (+   217us) catalog_manager.cc:2916] Committing in-memory state
1120 03:01:02.469445 (+1914218us) catalog_manager.cc:2920] Aborting table tasks
1120 03:01:02.507486 (+ 38041us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:00.346801 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:00.347217 (+   416us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:00.347241 (+    24us) write_op.cc:440] Acquired schema lock
1120 03:01:00.347253 (+    12us) tablet.cc:710] Decoding operations
1120 03:01:00.347559 (+   306us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:00.347596 (+    37us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:00.347617 (+    21us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:01:00.347896 (+   279us) tablet.cc:758] Row locks acquired
1120 03:01:00.347909 (+    13us) write_op.cc:260] PREPARE: finished
1120 03:01:00.348028 (+   119us) write_op.cc:270] Start()
1120 03:01:00.348103 (+    75us) write_op.cc:276] Timestamp: P: 1795143660347965 usec, L: 0
1120 03:01:00.348118 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:01:00.348613 (+   495us) log.cc:844] Serialized 725 byte log entry
1120 03:01:00.544387 (+195774us) op_driver.cc:466] REPLICATION: finished
1120 03:01:00.552875 (+  8488us) write_op.cc:301] APPLY: starting
1120 03:01:00.552928 (+    53us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:00.553033 (+   105us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:00.553134 (+   101us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:00.553387 (+   253us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:00.553529 (+   142us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
1120 03:01:00.553576 (+    47us) write_op.cc:312] APPLY: finished
1120 03:01:00.553749 (+   173us) log.cc:844] Serialized 29 byte log entry
1120 03:01:00.553884 (+   135us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:00.553944 (+    60us) write_op.cc:459] Released schema lock
1120 03:01:00.553974 (+    30us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:00.556531 (+  2557us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:00.556556 (+    25us) write_op.cc:459] Released schema lock
Metrics: {"mutex_wait_us":13720,"child_traces":[["op",{"apply.queue_time_us":8342,"apply.run_cpu_time_us":1305,"apply.run_wall_time_us":3614,"num_ops":2,"prepare.queue_time_us":33063,"prepare.run_cpu_time_us":2005,"prepare.run_wall_time_us":2006,"replication_time_us":196167}]]}
I20261120 03:01:02.500550  1181 consensus_queue.cc:261] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:02.471093   500 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:45640 (request call id 171) took 2706 ms. Trace:
W20261120 03:01:02.515734   498 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.67:53803 (request call id 21) took 1747 ms (client timeout 1000 ms). Trace:
I20261120 03:01:02.515795  1851 raft_consensus.cc:534] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
W20261120 03:01:02.516074   498 rpcz_store.cc:269] 1120 03:01:00.767967 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:00.768120 (+   153us) service_pool.cc:224] Handling call
1120 03:01:00.908194 (+140074us) spinlock_profiling.cc:245] Waited 137 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07d9ce 00007f155abf0174 00007f155abf0102 00007f155abf0031 00007f155abeffac 00007f155abef262 00007f1558e990bc 00007f1558e984f6
1120 03:01:02.515669 (+1607475us) inbound_call.cc:177] Queueing success response
Metrics: {"reported_tablets":22,"spinlock_wait_cycles":314404096}
I20261120 03:01:02.514592   500 rpcz_store.cc:276] 1120 03:00:59.765128 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:00.266236 (+501108us) service_pool.cc:224] Handling call
1120 03:01:00.267099 (+   863us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:01:00.267760 (+   661us) catalog_manager.cc:3769] Apply alter schema
1120 03:01:00.268338 (+   578us) catalog_manager.cc:3801] Acquired catalog manager lock
1120 03:01:00.268551 (+   213us) catalog_manager.cc:3972] Updating metadata on disk
1120 03:01:00.552270 (+283719us) catalog_manager.cc:4003] Committing alterations to in-memory state
1120 03:01:02.471037 (+1918767us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:00.273981 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:00.274414 (+   433us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:00.274440 (+    26us) write_op.cc:440] Acquired schema lock
1120 03:01:00.274454 (+    14us) tablet.cc:710] Decoding operations
1120 03:01:00.274716 (+   262us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:00.274754 (+    38us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:00.274777 (+    23us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:01:00.274939 (+   162us) tablet.cc:758] Row locks acquired
1120 03:01:00.274951 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:01:00.275061 (+   110us) write_op.cc:270] Start()
1120 03:01:00.275137 (+    76us) write_op.cc:276] Timestamp: P: 1795143660275017 usec, L: 0
1120 03:01:00.275154 (+    17us) op_driver.cc:350] REPLICATION: starting
1120 03:01:00.275614 (+   460us) log.cc:844] Serialized 405 byte log entry
1120 03:01:00.537885 (+262271us) op_driver.cc:466] REPLICATION: finished
1120 03:01:00.551039 (+ 13154us) write_op.cc:301] APPLY: starting
1120 03:01:00.551097 (+    58us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:00.551193 (+    96us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:00.551314 (+   121us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:00.551484 (+   170us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:00.551616 (+   132us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:01:00.551658 (+    42us) write_op.cc:312] APPLY: finished
1120 03:01:00.551839 (+   181us) log.cc:844] Serialized 23 byte log entry
1120 03:01:00.551987 (+   148us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:00.552049 (+    62us) write_op.cc:459] Released schema lock
1120 03:01:00.552078 (+    29us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:00.552605 (+   527us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:00.552630 (+    25us) write_op.cc:459] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":12988,"apply.run_cpu_time_us":1262,"apply.run_wall_time_us":1537,"num_ops":1,"prepare.queue_time_us":4293,"prepare.run_cpu_time_us":1800,"prepare.run_wall_time_us":1799,"replication_time_us":262634}]]}
I20261120 03:01:02.516229  1851 raft_consensus.cc:556] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:02.516872  1046 tablet_service.cc:1542] Processing DeleteTablet for tablet 88b9e9692a874ae993dde4b9e2586a70 with delete_type TABLET_DATA_DELETED (replaced by tablet 9e42a416be884341994d2d5395286798) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:02.516997   498 master_service.cc:965] ReplaceTablet: received request to replace tablet 29e6a50c9e1f4b778737abdef9ad073a from {username='slave'} at 127.0.0.1:45640
I20261120 03:01:02.471474   496 catalog_manager.cc:6409] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: default.Table_9fe8b8c914534e4ca5602de7c1b1d280 [id=f3b5f7a4e5494e82a472d0025df96e0c] alter complete (version 2)
W20261120 03:01:02.518586   496 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.66:36141 (request call id 30) took 2040 ms (client timeout 1000 ms). Trace:
I20261120 03:01:02.519025  1046 tablet_service.cc:1495] Processing CreateTablet for tablet ea98301805a740c4922b626927437a92 (DEFAULT_TABLE table=default.Table_664fa9b65c084e0ba9c360a594a48b66 [id=b2c3e78beca54c63bd72707806e38339]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.520131  1046 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ea98301805a740c4922b626927437a92. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.520500  1851 leader_election.cc:290] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
W20261120 03:01:02.521958  1011 leader_election.cc:343] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Not found: Tablet not found: 014bd9c83be34c31b90fe4bcf31f1649
I20261120 03:01:02.517287   500 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_736330745d7048c6b1fad681e15ba40d"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:02.522957  1050 tablet_service.cc:1542] Processing DeleteTablet for tablet 68179488b9bc4e03b65a9f1d434c95b5 with delete_type TABLET_DATA_DELETED (replaced by tablet 4d79fc4c8f1c4a5f83c0dc94337c857e) from {username='slave'} at 127.0.0.1:50744
W20261120 03:01:02.523505   494 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.67:53803 (request call id 20) took 2754 ms (client timeout 1000 ms). Trace:
W20261120 03:01:02.523821   494 rpcz_store.cc:269] 1120 03:00:59.769055 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.897336 (+128281us) service_pool.cc:224] Handling call
1120 03:01:02.523444 (+2626108us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":61237,"reported_tablets":25}
W20261120 03:01:02.524241   500 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_736330745d7048c6b1fad681e15ba40d in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:01:02.524384  1050 tablet_service.cc:1542] Processing DeleteTablet for tablet ad4e9e0de4954904842bfd5f15f530f0 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:01:00 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:02.524588   494 master_service.cc:965] ReplaceTablet: received request to replace tablet 0db7db25d779459ebde532f938493975 from {username='slave'} at 127.0.0.1:45640
W20261120 03:01:02.518878   496 rpcz_store.cc:269] 1120 03:01:00.478244 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:00.478341 (+    97us) service_pool.cc:224] Handling call
1120 03:01:00.540104 (+ 61763us) spinlock_profiling.cc:245] Waited 59.6 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07d9ce 00007f155abf0174 00007f155abf0102 00007f155abf0031 00007f155abeffac 00007f155abef262 00007f1558e990bc 00007f1558e984f6
1120 03:01:02.518522 (+1978418us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:01.316687 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:01.317062 (+   375us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:01.317084 (+    22us) write_op.cc:440] Acquired schema lock
1120 03:01:01.317096 (+    12us) tablet.cc:710] Decoding operations
1120 03:01:01.317382 (+   286us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:01.317418 (+    36us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:01.317438 (+    20us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:01:01.317577 (+   139us) tablet.cc:758] Row locks acquired
1120 03:01:01.317590 (+    13us) write_op.cc:260] PREPARE: finished
1120 03:01:01.317694 (+   104us) write_op.cc:270] Start()
1120 03:01:01.317774 (+    80us) write_op.cc:276] Timestamp: P: 1795143661317664 usec, L: 0
1120 03:01:01.317789 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:01:01.318260 (+   471us) log.cc:844] Serialized 527 byte log entry
1120 03:01:01.370449 (+ 52189us) op_driver.cc:466] REPLICATION: finished
1120 03:01:01.371118 (+   669us) write_op.cc:301] APPLY: starting
1120 03:01:01.371169 (+    51us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:01.371254 (+    85us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:01.371344 (+    90us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:01.371484 (+   140us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:01.371598 (+   114us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:01:01.371639 (+    41us) write_op.cc:312] APPLY: finished
1120 03:01:01.371788 (+   149us) log.cc:844] Serialized 23 byte log entry
1120 03:01:01.371921 (+   133us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:01.371978 (+    57us) write_op.cc:459] Released schema lock
1120 03:01:01.372007 (+    29us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:01.372211 (+   204us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:01.372234 (+    23us) write_op.cc:459] Released schema lock
Related trace 'op':
1120 03:01:01.389575 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:01.390004 (+   429us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:01.390029 (+    25us) write_op.cc:440] Acquired schema lock
1120 03:01:01.390043 (+    14us) tablet.cc:710] Decoding operations
1120 03:01:01.390337 (+   294us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:01.390375 (+    38us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:01.390396 (+    21us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:01:01.390540 (+   144us) tablet.cc:758] Row locks acquired
1120 03:01:01.390553 (+    13us) write_op.cc:260] PREPARE: finished
1120 03:01:01.390675 (+   122us) write_op.cc:270] Start()
1120 03:01:01.390757 (+    82us) write_op.cc:276] Timestamp: P: 1795143661390637 usec, L: 0
1120 03:01:01.390773 (+    16us) op_driver.cc:350] REPLICATION: starting
1120 03:01:01.391242 (+   469us) log.cc:844] Serialized 356 byte log entry
1120 03:01:01.419709 (+ 28467us) op_driver.cc:466] REPLICATION: finished
1120 03:01:01.423438 (+  3729us) write_op.cc:301] APPLY: starting
1120 03:01:01.423492 (+    54us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:01.423586 (+    94us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:01.423693 (+   107us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:01.423845 (+   152us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:01.423967 (+   122us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:01:01.424008 (+    41us) write_op.cc:312] APPLY: finished
1120 03:01:01.424204 (+   196us) log.cc:844] Serialized 23 byte log entry
1120 03:01:01.424340 (+   136us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:01.424398 (+    58us) write_op.cc:459] Released schema lock
1120 03:01:01.424431 (+    33us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:01.424654 (+   223us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:01.424677 (+    23us) write_op.cc:459] Released schema lock
Metrics: {"mutex_wait_us":7895,"reported_tablets":7,"spinlock_wait_cycles":137045376,"child_traces":[["op",{"apply.queue_time_us":530,"apply.run_cpu_time_us":1060,"apply.run_wall_time_us":1060,"num_ops":1,"prepare.queue_time_us":134584,"prepare.run_cpu_time_us":1911,"prepare.run_wall_time_us":39721,"raft.queue_time_us":194,"raft.run_cpu_time_us":673,"raft.run_wall_time_us":673,"replication_time_us":52564}],["op",{"apply.queue_time_us":3567,"apply.run_cpu_time_us":1185,"apply.run_wall_time_us":1182,"num_ops":1,"prepare.queue_time_us":11821,"prepare.run_cpu_time_us":1877,"prepare.run_wall_time_us":1878,"replication_time_us":28842}]]}
I20261120 03:01:02.528331  1050 tablet_service.cc:1542] Processing DeleteTablet for tablet a08aa5ddcf1a4061a70275a8eecb1856 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:53 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:02.529456   496 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_4cf0ca76843b4dc3982bdaee328cb161"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:01:02.531656   496 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_4cf0ca76843b4dc3982bdaee328cb161 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:01:02.532137   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 84204.5us
W20261120 03:01:02.494493   502 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.66:36141 (request call id 31) took 992 ms (client timeout 1000 ms). Trace:
W20261120 03:01:02.532644   502 rpcz_store.cc:269] 1120 03:01:01.502139 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:01.630796 (+128657us) service_pool.cc:224] Handling call
1120 03:01:02.494434 (+863638us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:01.869848 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:01.870259 (+   411us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:01.870285 (+    26us) write_op.cc:440] Acquired schema lock
1120 03:01:01.870299 (+    14us) tablet.cc:710] Decoding operations
1120 03:01:01.870744 (+   445us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:01.870787 (+    43us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:01.870810 (+    23us) tablet.cc:742] Acquiring locks for 3 operations
1120 03:01:01.871073 (+   263us) tablet.cc:758] Row locks acquired
1120 03:01:01.871085 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:01:01.871207 (+   122us) write_op.cc:270] Start()
1120 03:01:01.871289 (+    82us) write_op.cc:276] Timestamp: P: 1795143661871152 usec, L: 0
1120 03:01:01.871304 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:01:01.871792 (+   488us) log.cc:844] Serialized 1275 byte log entry
1120 03:01:01.892580 (+ 20788us) op_driver.cc:466] REPLICATION: finished
1120 03:01:01.892946 (+   366us) write_op.cc:301] APPLY: starting
1120 03:01:01.893001 (+    55us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:01.893138 (+   137us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:01.893245 (+   107us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:01.893567 (+   322us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:01.893709 (+   142us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=3
1120 03:01:01.893752 (+    43us) write_op.cc:312] APPLY: finished
1120 03:01:01.893942 (+   190us) log.cc:844] Serialized 35 byte log entry
1120 03:01:01.894102 (+   160us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:01.894173 (+    71us) write_op.cc:459] Released schema lock
1120 03:01:01.894204 (+    31us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:01.894414 (+   210us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:01.894437 (+    23us) write_op.cc:459] Released schema lock
Metrics: {"mutex_wait_us":3444,"reported_tablets":17,"child_traces":[["op",{"apply.queue_time_us":197,"apply.run_cpu_time_us":1438,"apply.run_wall_time_us":1438,"num_ops":3,"prepare.queue_time_us":209,"prepare.run_cpu_time_us":2273,"prepare.run_wall_time_us":2273,"raft.queue_time_us":1560,"raft.run_cpu_time_us":1588,"raft.run_wall_time_us":1588,"replication_time_us":21175}]]}
I20261120 03:01:02.536355   502 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_555e4ecfadce486baefa8c3e2d1bc272" } modify_external_catalogs: true
I20261120 03:01:02.536962   502 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_555e4ecfadce486baefa8c3e2d1bc272" } modify_external_catalogs: true
I20261120 03:01:02.540354  1181 ts_tablet_manager.cc:1480] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.164s	user 0.005s	sys 0.001s
I20261120 03:01:02.481932   773 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "091dd73d27df4db09ab750747da7584e" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:02.500885   751 tablet_service.cc:1495] Processing CreateTablet for tablet 1d00f5c083c44b47bbf004b1c5dfcc1f (DEFAULT_TABLE table=default.Table_58015d78e3d24cd79830293a3ce6176c [id=49c74781c4bd4acf962012a77f3d3b1b]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.553959   751 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1d00f5c083c44b47bbf004b1c5dfcc1f. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.505571   752 tablet_service.cc:1495] Processing CreateTablet for tablet f8f2fc38086d4e47b1664a23c0cfee21 (DEFAULT_TABLE table=default.Table_184b1c05e67e47718e9f58e229c4c748 [id=e35d83b51ea745ab96033865032acf86]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.511219   754 tablet_service.cc:1542] Processing DeleteTablet for tablet ad4e9e0de4954904842bfd5f15f530f0 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:01:00 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:01:02.514981   757 tablet_service.cc:1542] Processing DeleteTablet for tablet 5c5c0ac869a1419fad9f61ec647027ee with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:53 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:01:02.429967  1324 raft_consensus.cc:366] T 4da5eb2f2f114cc792488c5c74bb1414 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.559137   752 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f8f2fc38086d4e47b1664a23c0cfee21. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.559190  1324 raft_consensus.cc:392] T 4da5eb2f2f114cc792488c5c74bb1414 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:02.559525  1324 raft_consensus.cc:787] T 4da5eb2f2f114cc792488c5c74bb1414 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
W20261120 03:01:02.561877   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet a6ae8007455d4be3b50672b9c6b0fd8f because the tablet was not found. No further retry: Not found: Tablet not found: a6ae8007455d4be3b50672b9c6b0fd8f
W20261120 03:01:02.563455   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet ce612f9a121b4b18908724d2726e156f because the tablet was not found. No further retry: Not found: Tablet not found: ce612f9a121b4b18908724d2726e156f
W20261120 03:01:02.564026   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 4abb67f3819a45b9b05dde04842e2c9d because the tablet was not found. No further retry: Not found: Tablet not found: 4abb67f3819a45b9b05dde04842e2c9d
W20261120 03:01:02.564589   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 020be7f7cda0467aab56f657cbe8d0ad because the tablet was not found. No further retry: Not found: Tablet not found: 020be7f7cda0467aab56f657cbe8d0ad
W20261120 03:01:02.565112   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet a1192a7f050a4f5387a30b2216dec5f0 because the tablet was not found. No further retry: Not found: Tablet not found: a1192a7f050a4f5387a30b2216dec5f0
W20261120 03:01:02.565639   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 200b234da897419a82bf5844620adf24 because the tablet was not found. No further retry: Not found: Tablet not found: 200b234da897419a82bf5844620adf24
I20261120 03:01:02.567726  1181 tablet_bootstrap.cc:492] T 9dadc05b726346309d6b350f9abb4abb P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:02.570436  1324 consensus_queue.cc:261] T 4da5eb2f2f114cc792488c5c74bb1414 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.575790   495 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_e53618d9c5b7456f9e23d36a5f55affe"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:01:02.576561   493 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.65:40295 (request call id 35) took 2931 ms (client timeout 1000 ms). Trace:
W20261120 03:01:02.581365   495 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_e53618d9c5b7456f9e23d36a5f55affe in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:01:02.572431  1886 leader_election.cc:290] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Requested vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:02.585824  1324 ts_tablet_manager.cc:1480] T 4da5eb2f2f114cc792488c5c74bb1414 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.158s	user 0.002s	sys 0.005s
I20261120 03:01:02.586715  1324 tablet_bootstrap.cc:492] T 26f8ba69516c447dbd91e0d87cd519d3 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:01:02.588428   902 tablet_service.cc:1542] Processing DeleteTablet for tablet a1192a7f050a4f5387a30b2216dec5f0 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:01:00 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:01:02.589365  1796 leader_election.cc:290] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:02.590943   900 tablet_service.cc:1542] Processing DeleteTablet for tablet b549a06d15db449b851fe9549a963416 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:54 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:01:02.594478   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "091dd73d27df4db09ab750747da7584e" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:02.599581   896 tablet_service.cc:1542] Processing DeleteTablet for tablet ad4e9e0de4954904842bfd5f15f530f0 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:01:00 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:01:02.600421   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3bc0f2f9cb7945ab9b0ead61c934f9dd" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:01:02.601161   918 raft_consensus.cc:2628] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 5beeefe8bef54caeac463d05d8c30fa6 in current term 1: Already voted for candidate 3d30c2428730481db9b0ffd279dd3440 in this term.
I20261120 03:01:02.602901   895 tablet_service.cc:1495] Processing CreateTablet for tablet f8f2fc38086d4e47b1664a23c0cfee21 (DEFAULT_TABLE table=default.Table_184b1c05e67e47718e9f58e229c4c748 [id=e35d83b51ea745ab96033865032acf86]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.604584   895 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f8f2fc38086d4e47b1664a23c0cfee21. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.606508   900 tablet_service.cc:1542] Processing DeleteTablet for tablet c6a65c1a4a574961bec657ad7c9048e0 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:57 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:01:02.614121  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3bc0f2f9cb7945ab9b0ead61c934f9dd" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:01:02.616282   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet a08aa5ddcf1a4061a70275a8eecb1856 (table default.Table_3c79e29085ad4875874397eaefcab250 [id=47e54c07e54d40adb2ac6907cb03bb68]) successfully deleted
W20261120 03:01:02.622836   709 leader_election.cc:343] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:02.623225   709 leader_election.cc:304] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:02.587041  1761 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:52644 (request call id 137) took 2707 ms. Trace:
I20261120 03:01:02.623812  1761 rpcz_store.cc:276] 1120 03:00:59.880068 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.880596 (+   528us) service_pool.cc:224] Handling call
1120 03:01:02.586716 (+2706120us) ts_tablet_manager.cc:1197] Deleting tablet a6ae8007455d4be3b50672b9c6b0fd8f
1120 03:01:02.586768 (+    52us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:02.586975 (+   207us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":2704209}
I20261120 03:01:02.624923  1886 raft_consensus.cc:2966] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20261120 03:01:02.625152  1761 tablet_replica.cc:333] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:01:02.625970  1761 raft_consensus.cc:2440] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:02.626700  1761 raft_consensus.cc:2469] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:02.628991   498 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 29e6a50c9e1f4b778737abdef9ad073a
I20261120 03:01:02.631003   498 catalog_manager.cc:6952] ReplaceTablet: tablet 29e6a50c9e1f4b778737abdef9ad073a deleted and replaced by tablet 81d7cb5526944095b7054464971e8fa4
I20261120 03:01:02.633352   498 rpcz_store.cc:275] Call kudu.master.MasterService.GetMasterRegistration from 127.0.0.1:54542 (request call id 1) took 1707 ms. Trace:
I20261120 03:01:02.633754   498 rpcz_store.cc:276] 1120 03:01:00.926311 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:02.632805 (+1706494us) service_pool.cc:224] Handling call
1120 03:01:02.633275 (+   470us) inbound_call.cc:177] Queueing success response
Metrics: {}
I20261120 03:01:02.660198  1761 ts_tablet_manager.cc:1962] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:02.665472  1923 tablet.cc:1872] T daae644853734cd5818a46ecaa5ce7df P 823abdbd90ba47bbacad0f8dcff78234: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20261120 03:01:02.643868  1181 tablet_bootstrap.cc:654] T 9dadc05b726346309d6b350f9abb4abb P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:02.674824  1012 leader_election.cc:304] T 091dd73d27df4db09ab750747da7584e P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:01:02.676527  1893 raft_consensus.cc:2966] T 091dd73d27df4db09ab750747da7584e P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:02.684995  1847 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
I20261120 03:01:02.695559   900 tablet_service.cc:1495] Processing CreateTablet for tablet ea98301805a740c4922b626927437a92 (DEFAULT_TABLE table=default.Table_664fa9b65c084e0ba9c360a594a48b66 [id=b2c3e78beca54c63bd72707806e38339]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.696895   900 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ea98301805a740c4922b626927437a92. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:01:02.707844  1012 leader_election.cc:343] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Not found: Tablet not found: 014bd9c83be34c31b90fe4bcf31f1649
I20261120 03:01:02.708269   898 tablet_service.cc:1542] Processing DeleteTablet for tablet 29e6a50c9e1f4b778737abdef9ad073a with delete_type TABLET_DATA_DELETED (replaced by tablet 81d7cb5526944095b7054464971e8fa4) from {username='slave'} at 127.0.0.1:44896
I20261120 03:01:02.708278  1012 leader_election.cc:304] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:01:02.709427  1893 raft_consensus.cc:2966] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:02.712402   898 tablet_service.cc:1495] Processing CreateTablet for tablet ad4e9e0de4954904842bfd5f15f530f0 (DEFAULT_TABLE table=default.Table_646d629dfeea413a86bc3edf68c7d6ae [id=aa0b779e8ea84d928af40806a7913f1a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.713406   898 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ad4e9e0de4954904842bfd5f15f530f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.716061   896 tablet_service.cc:1495] Processing CreateTablet for tablet 1d8c0a9abc7a42c8927a617491e028aa (DEFAULT_TABLE table=default.Table_ee81fc6028814ea3a0cf91abf82d2feb [id=1b5aeef45a0246728133347b90dfda61]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.717154   896 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1d8c0a9abc7a42c8927a617491e028aa. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:01:02.792027   488 catalog_manager.cc:4863] Async tablet task AlterTable RPC for tablet 73c993944b0a4977b55d40ad63bb69e3 (table default.Table_a96b0ac352c545da9b955502704e266b [id=1cdd4bbccdbf4449ba67b9aa7089a6fd], current schema version=1) failed: Not found: failed to reset TS proxy: No leader found: Tablet 73c993944b0a4977b55d40ad63bb69e3 consensus state has no leader
W20261120 03:01:02.806911   710 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 72445.9us
I20261120 03:01:02.811018  1809 raft_consensus.cc:534] T 4da5eb2f2f114cc792488c5c74bb1414 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:02.818819   750 tablet_service.cc:1542] Processing DeleteTablet for tablet 4abb67f3819a45b9b05dde04842e2c9d with delete_type TABLET_DATA_DELETED (replaced by tablet df90c4151aa44b83a12b17680dc161da) from {username='slave'} at 127.0.0.1:52644
I20261120 03:01:02.819175   756 tablet_service.cc:1542] Processing DeleteTablet for tablet 020be7f7cda0467aab56f657cbe8d0ad with delete_type TABLET_DATA_DELETED (replaced by tablet 600a18b7e5104a78935d906b8adfde6f) from {username='slave'} at 127.0.0.1:52644
I20261120 03:01:02.820798  1324 tablet_bootstrap.cc:654] T 26f8ba69516c447dbd91e0d87cd519d3 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:02.821075   756 tablet_service.cc:1495] Processing CreateTablet for tablet ea98301805a740c4922b626927437a92 (DEFAULT_TABLE table=default.Table_664fa9b65c084e0ba9c360a594a48b66 [id=b2c3e78beca54c63bd72707806e38339]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.824172   756 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ea98301805a740c4922b626927437a92. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.811501  1809 raft_consensus.cc:556] T 4da5eb2f2f114cc792488c5c74bb1414 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:02.843493   750 tablet_service.cc:1495] Processing CreateTablet for tablet ad4e9e0de4954904842bfd5f15f530f0 (DEFAULT_TABLE table=default.Table_646d629dfeea413a86bc3edf68c7d6ae [id=aa0b779e8ea84d928af40806a7913f1a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.844657   750 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ad4e9e0de4954904842bfd5f15f530f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.858131  1809 leader_election.cc:290] T 4da5eb2f2f114cc792488c5c74bb1414 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
W20261120 03:01:02.867149   710 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 59189.4us
I20261120 03:01:02.876814   923 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "4da5eb2f2f114cc792488c5c74bb1414" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:01:02.887472   710 leader_election.cc:343] T 4da5eb2f2f114cc792488c5c74bb1414 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:01:02.896679  1837 thread.cc:642] prepare [worker] (thread pool) Time spent creating pthread: real 0.512s	user 0.008s	sys 0.017s
W20261120 03:01:02.897209  1837 thread.cc:609] prepare [worker] (thread pool) Time spent starting thread: real 0.513s	user 0.009s	sys 0.017s
W20261120 03:01:02.898837  1876 thread.cc:642] wal-append [worker] (thread pool) Time spent creating pthread: real 0.507s	user 0.003s	sys 0.017s
W20261120 03:01:02.899246  1876 thread.cc:609] wal-append [worker] (thread pool) Time spent starting thread: real 0.508s	user 0.003s	sys 0.017s
I20261120 03:01:02.904663  1180 tablet_bootstrap.cc:654] T ad49c8e00aca4a88a091baceae320477 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:02.888859  1920 tablet.cc:1872] T daae644853734cd5818a46ecaa5ce7df P 5beeefe8bef54caeac463d05d8c30fa6: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20261120 03:01:02.927469  1918 tablet_replica.cc:333] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:01:02.928509  1918 raft_consensus.cc:2440] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:02.929093  1918 raft_consensus.cc:2469] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
W20261120 03:01:02.931663   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 398889us
I20261120 03:01:02.933285  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3bc0f2f9cb7945ab9b0ead61c934f9dd" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:01:02.955518   498 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_3d732539a5434ae193267b36578eb1c3" } new_table_name: "default.Table_12409ca618284aa285f193bbe6547101" modify_external_catalogs: true
W20261120 03:01:02.576865   493 rpcz_store.cc:269] 1120 03:00:59.645269 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.844353 (+199084us) service_pool.cc:224] Handling call
1120 03:01:02.576486 (+2732133us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":151882,"reported_tablets":38,"spinlock_wait_cycles":14720}
I20261120 03:01:02.977260  1045 tablet_service.cc:1495] Processing CreateTablet for tablet ad4e9e0de4954904842bfd5f15f530f0 (DEFAULT_TABLE table=default.Table_646d629dfeea413a86bc3edf68c7d6ae [id=aa0b779e8ea84d928af40806a7913f1a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:02.978336  1045 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ad4e9e0de4954904842bfd5f15f530f0. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:02.983369  1046 tablet_service.cc:1542] Processing DeleteTablet for tablet 29e6a50c9e1f4b778737abdef9ad073a with delete_type TABLET_DATA_DELETED (replaced by tablet 81d7cb5526944095b7054464971e8fa4) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:02.991297   773 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3bc0f2f9cb7945ab9b0ead61c934f9dd" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:02.992015   773 raft_consensus.cc:2628] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 5beeefe8bef54caeac463d05d8c30fa6 in this term.
I20261120 03:01:02.996492   756 tablet_service.cc:1542] Processing DeleteTablet for tablet 29e6a50c9e1f4b778737abdef9ad073a with delete_type TABLET_DATA_DELETED (replaced by tablet 81d7cb5526944095b7054464971e8fa4) from {username='slave'} at 127.0.0.1:52644
I20261120 03:01:03.007855  1791 raft_consensus.cc:534] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:03.008306  1791 raft_consensus.cc:556] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.014679  1791 leader_election.cc:290] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:03.028127  1761 ts_tablet_manager.cc:1975] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:03.042899  1046 tablet_service.cc:1495] Processing CreateTablet for tablet 1d8c0a9abc7a42c8927a617491e028aa (DEFAULT_TABLE table=default.Table_ee81fc6028814ea3a0cf91abf82d2feb [id=1b5aeef45a0246728133347b90dfda61]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.044234  1046 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1d8c0a9abc7a42c8927a617491e028aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.063364  1936 raft_consensus.cc:534] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 18331bef5c884dcab9dbd26bc2c82273)
I20261120 03:01:03.063993  1936 raft_consensus.cc:556] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
I20261120 03:01:03.066890   435 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" candidate_term: 4 candidate_status { last_received { term: 3 index: 228 } } ignore_live_leader: false dest_uuid: "e102e07459954b4ea13b65ab9fd5d18b" is_pre_election: true
I20261120 03:01:03.078550  1936 leader_election.cc:290] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers e102e07459954b4ea13b65ab9fd5d18b (127.0.96.126:44467), 18331bef5c884dcab9dbd26bc2c82273 (127.0.96.125:37411)
I20261120 03:01:03.097106  1181 tablet_bootstrap.cc:492] T 9dadc05b726346309d6b350f9abb4abb P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:03.097060  1837 raft_consensus.cc:534] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:03.097512  1181 ts_tablet_manager.cc:1449] T 9dadc05b726346309d6b350f9abb4abb P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.530s	user 0.053s	sys 0.051s
I20261120 03:01:03.097779  1837 raft_consensus.cc:556] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.100215  1181 raft_consensus.cc:366] T 9dadc05b726346309d6b350f9abb4abb P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:03.100803  1181 raft_consensus.cc:392] T 9dadc05b726346309d6b350f9abb4abb P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:03.101058  1181 raft_consensus.cc:787] T 9dadc05b726346309d6b350f9abb4abb P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:03.101752  1181 consensus_queue.cc:261] T 9dadc05b726346309d6b350f9abb4abb P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:03.103658  1181 ts_tablet_manager.cc:1480] T 9dadc05b726346309d6b350f9abb4abb P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.000s
I20261120 03:01:03.104498  1181 tablet_bootstrap.cc:492] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
W20261120 03:01:03.108755   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 1885397e42774a89bb347bc776284bc0 because the tablet was not found. No further retry: Not found: Tablet not found: 1885397e42774a89bb347bc776284bc0
W20261120 03:01:03.109398   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 88b9e9692a874ae993dde4b9e2586a70 because the tablet was not found. No further retry: Not found: Tablet not found: 88b9e9692a874ae993dde4b9e2586a70
I20261120 03:01:03.115253   512 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" candidate_term: 4 candidate_status { last_received { term: 3 index: 228 } } ignore_live_leader: false dest_uuid: "18331bef5c884dcab9dbd26bc2c82273" is_pre_election: true
I20261120 03:01:03.116796  1645 leader_election.cc:304] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2599fd9eef7d4ce990c40e2356a37c3f; no voters: 18331bef5c884dcab9dbd26bc2c82273, e102e07459954b4ea13b65ab9fd5d18b
I20261120 03:01:03.117565  1936 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 2599fd9eef7d4ce990c40e2356a37c3f [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20261120 03:01:03.110838  1837 leader_election.cc:290] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
W20261120 03:01:03.114405   857 leader_election.cc:343] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:01:03.126595   857 consensus_peers.cc:597] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 -> Peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Couldn't send request to peer 823abdbd90ba47bbacad0f8dcff78234. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: INITIALIZED. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:01:03.128257  1075 raft_consensus.cc:3296] T 091dd73d27df4db09ab750747da7584e P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:03.147512  1918 ts_tablet_manager.cc:1962] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
W20261120 03:01:03.074085   709 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 243077us
I20261120 03:01:03.153481  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "4da5eb2f2f114cc792488c5c74bb1414" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:03.156992  1180 tablet_bootstrap.cc:492] T ad49c8e00aca4a88a091baceae320477 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:01:03.157402  1180 ts_tablet_manager.cc:1449] T ad49c8e00aca4a88a091baceae320477 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.750s	user 0.009s	sys 0.000s
I20261120 03:01:03.158556  1808 raft_consensus.cc:534] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:03.159036  1808 raft_consensus.cc:556] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.161582  1808 leader_election.cc:290] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
W20261120 03:01:03.109970   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 68179488b9bc4e03b65a9f1d434c95b5 because the tablet was not found. No further retry: Not found: Tablet not found: 68179488b9bc4e03b65a9f1d434c95b5
W20261120 03:01:03.166870   486 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.DeleteTablet -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 56888.9us
W20261120 03:01:03.167829   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet ad4e9e0de4954904842bfd5f15f530f0 because the tablet was not found. No further retry: Not found: Tablet not found: ad4e9e0de4954904842bfd5f15f530f0
I20261120 03:01:03.176110  1180 raft_consensus.cc:366] T ad49c8e00aca4a88a091baceae320477 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.076076  1324 tablet_bootstrap.cc:492] T 26f8ba69516c447dbd91e0d87cd519d3 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:01:03.078496   750 tablet_service.cc:1495] Processing CreateTablet for tablet 1d8c0a9abc7a42c8927a617491e028aa (DEFAULT_TABLE table=default.Table_ee81fc6028814ea3a0cf91abf82d2feb [id=1b5aeef45a0246728133347b90dfda61]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.176779  1180 raft_consensus.cc:392] T ad49c8e00aca4a88a091baceae320477 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:03.177012  1180 raft_consensus.cc:787] T ad49c8e00aca4a88a091baceae320477 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:03.177557   750 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1d8c0a9abc7a42c8927a617491e028aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.186823   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9ea5f0b680bd401bbc2b69d780e1ef50" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:03.190825  1324 ts_tablet_manager.cc:1449] T 26f8ba69516c447dbd91e0d87cd519d3 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.604s	user 0.013s	sys 0.003s
I20261120 03:01:03.196650   385 master-stress-test.cc:579] Killing test thread 1/10
I20261120 03:01:03.196960  1939 raft_consensus.cc:534] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 3 FOLLOWER]: Starting pre-election (detected failure of leader 18331bef5c884dcab9dbd26bc2c82273)
I20261120 03:01:03.197383  1939 raft_consensus.cc:556] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e102e07459954b4ea13b65ab9fd5d18b" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 44467 } } peers { permanent_uuid: "18331bef5c884dcab9dbd26bc2c82273" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 37411 } } peers { permanent_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 38133 } }
W20261120 03:01:03.198545   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet a08aa5ddcf1a4061a70275a8eecb1856 because the tablet was not found. No further retry: Not found: Tablet not found: a08aa5ddcf1a4061a70275a8eecb1856
I20261120 03:01:03.200060   512 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e102e07459954b4ea13b65ab9fd5d18b" candidate_term: 4 candidate_status { last_received { term: 3 index: 233 } } ignore_live_leader: false dest_uuid: "18331bef5c884dcab9dbd26bc2c82273" is_pre_election: true
I20261120 03:01:03.201112  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9ea5f0b680bd401bbc2b69d780e1ef50" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:03.208539  1939 leader_election.cc:290] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 18331bef5c884dcab9dbd26bc2c82273 (127.0.96.125:37411), 2599fd9eef7d4ce990c40e2356a37c3f (127.0.96.124:38133)
W20261120 03:01:03.209371   710 leader_election.cc:343] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:03.215467  1941 tablet.cc:1872] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20261120 03:01:03.218909  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00f7c7d95bae4d55be6b437701989ba3" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:01:03.200222   486 catalog_manager.cc:5061] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 29e6a50c9e1f4b778737abdef9ad073a because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 29e6a50c9e1f4b778737abdef9ad073a already in progress: creating tablet
W20261120 03:01:03.224195   486 catalog_manager.cc:5048] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet a6ae8007455d4be3b50672b9c6b0fd8f because the tablet was not found. No further retry: Not found: Tablet not found: a6ae8007455d4be3b50672b9c6b0fd8f
I20261120 03:01:03.150033  1761 log.cc:1199] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/88b9e9692a874ae993dde4b9e2586a70
I20261120 03:01:03.231531   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00f7c7d95bae4d55be6b437701989ba3" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:03.232239   777 raft_consensus.cc:2062] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:01:03.237202  1937 tablet.cc:1872] T daae644853734cd5818a46ecaa5ce7df P 3d30c2428730481db9b0ffd279dd3440: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20261120 03:01:03.251207  1181 tablet_bootstrap.cc:654] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:03.257364  1699 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e102e07459954b4ea13b65ab9fd5d18b" candidate_term: 4 candidate_status { last_received { term: 3 index: 233 } } ignore_live_leader: false dest_uuid: "2599fd9eef7d4ce990c40e2356a37c3f" is_pre_election: true
I20261120 03:01:03.257902  1761 ts_tablet_manager.cc:1996] T 88b9e9692a874ae993dde4b9e2586a70 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:01:03.259449   411 leader_election.cc:304] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e102e07459954b4ea13b65ab9fd5d18b; no voters: 18331bef5c884dcab9dbd26bc2c82273, 2599fd9eef7d4ce990c40e2356a37c3f
I20261120 03:01:03.260468  1939 raft_consensus.cc:2966] T 00000000000000000000000000000000 P e102e07459954b4ea13b65ab9fd5d18b [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20261120 03:01:03.263468  1324 raft_consensus.cc:366] T 26f8ba69516c447dbd91e0d87cd519d3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.264119  1324 raft_consensus.cc:392] T 26f8ba69516c447dbd91e0d87cd519d3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:03.264372  1324 raft_consensus.cc:787] T 26f8ba69516c447dbd91e0d87cd519d3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:01:03.265044  1324 consensus_queue.cc:261] T 26f8ba69516c447dbd91e0d87cd519d3 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.177611  1180 consensus_queue.cc:261] T ad49c8e00aca4a88a091baceae320477 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.269764  1180 ts_tablet_manager.cc:1480] T ad49c8e00aca4a88a091baceae320477 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.112s	user 0.007s	sys 0.000s
W20261120 03:01:03.284556   709 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 50763.3us
W20261120 03:01:03.285202   709 leader_election.cc:343] T 4da5eb2f2f114cc792488c5c74bb1414 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:03.285571   709 leader_election.cc:304] T 4da5eb2f2f114cc792488c5c74bb1414 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:03.286967  1930 raft_consensus.cc:2966] T 4da5eb2f2f114cc792488c5c74bb1414 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20261120 03:01:03.287030   709 leader_election.cc:343] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:03.287364   709 leader_election.cc:304] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:03.302484  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ba85dab0d238487181ce5c1592a2c7fd" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
W20261120 03:01:03.305356   486 outbound_call.cc:321] RPC callback for RPC call kudu.tserver.TabletServerAdminService.CreateTablet -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 79530.2us
I20261120 03:01:03.309614  1180 tablet_bootstrap.cc:492] T 73c993944b0a4977b55d40ad63bb69e3 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:01:03.320204  1181 tablet_bootstrap.cc:492] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:03.320642  1181 ts_tablet_manager.cc:1449] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.216s	user 0.009s	sys 0.002s
I20261120 03:01:03.321949   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ba85dab0d238487181ce5c1592a2c7fd" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:03.322655   777 raft_consensus.cc:2062] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:01:03.322978  1181 raft_consensus.cc:366] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:03.323576  1181 raft_consensus.cc:392] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:03.323901  1181 raft_consensus.cc:787] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:03.338773  1808 raft_consensus.cc:2966] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:03.338682  1181 consensus_queue.cc:261] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:03.340611  1181 ts_tablet_manager.cc:1480] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.020s	user 0.005s	sys 0.000s
I20261120 03:01:03.340762  1761 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:52644 (request call id 133) took 3462 ms. Trace:
I20261120 03:01:03.341454  1181 tablet_bootstrap.cc:492] T 59cc37f7ef9c409493bd89d0d2097339 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:03.341039  1761 rpcz_store.cc:276] 1120 03:00:59.878793 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.882430 (+  3637us) service_pool.cc:224] Handling call
1120 03:01:02.624988 (+2742558us) ts_tablet_manager.cc:1197] Deleting tablet 88b9e9692a874ae993dde4b9e2586a70
1120 03:01:02.625051 (+    63us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:02.839814 (+214763us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:03.027999 (+188185us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:03.340711 (+312712us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":81898,"tablet-delete.queue_time_us":2742095}
I20261120 03:01:03.342810  1761 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:52644 (request call id 134) took 3463 ms. Trace:
I20261120 03:01:03.343094  1761 rpcz_store.cc:276] 1120 03:00:59.879122 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.883577 (+  4455us) service_pool.cc:224] Handling call
1120 03:01:03.342557 (+3458980us) ts_tablet_manager.cc:1197] Deleting tablet 68179488b9bc4e03b65a9f1d434c95b5
1120 03:01:03.342603 (+    46us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:03.342763 (+   160us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3458483}
I20261120 03:01:03.344031  1761 tablet_replica.cc:333] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:01:03.344745  1761 raft_consensus.cc:2440] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 2 LEADER]: Raft consensus shutting down.
I20261120 03:01:03.345625  1761 raft_consensus.cc:2469] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 2 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:03.327272   857 leader_election.cc:304] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
W20261120 03:01:03.347775   709 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 60763.5us
I20261120 03:01:03.349689   496 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:45640 (request call id 188) took 1516 ms. Trace:
I20261120 03:01:03.350016   496 rpcz_store.cc:276] 1120 03:01:01.833407 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:02.529067 (+695660us) service_pool.cc:224] Handling call
1120 03:01:02.532094 (+  3027us) catalog_manager.cc:2145] Acquired catalog manager lock
1120 03:01:02.551112 (+ 19018us) catalog_manager.cc:2191] Created new table and tablet info
1120 03:01:03.348831 (+797719us) catalog_manager.cc:2251] Wrote table and tablets to system table
1120 03:01:03.349286 (+   455us) catalog_manager.cc:2285] Inserted table and tablets into CatalogManager maps
1120 03:01:03.349607 (+   321us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:02.553276 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:02.553666 (+   390us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:02.553688 (+    22us) write_op.cc:440] Acquired schema lock
1120 03:01:02.553699 (+    11us) tablet.cc:710] Decoding operations
1120 03:01:02.553962 (+   263us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:02.553996 (+    34us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:02.554018 (+    22us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:01:02.554255 (+   237us) tablet.cc:758] Row locks acquired
1120 03:01:02.554267 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:01:02.554374 (+   107us) write_op.cc:270] Start()
1120 03:01:02.554446 (+    72us) write_op.cc:276] Timestamp: P: 1795143662554311 usec, L: 0
1120 03:01:02.554461 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:01:02.554991 (+   530us) log.cc:844] Serialized 433 byte log entry
1120 03:01:03.346488 (+791497us) op_driver.cc:466] REPLICATION: finished
1120 03:01:03.347361 (+   873us) write_op.cc:301] APPLY: starting
1120 03:01:03.347419 (+    58us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:03.347530 (+   111us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:03.347629 (+    99us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:03.348021 (+   392us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:03.348149 (+   128us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
1120 03:01:03.348192 (+    43us) write_op.cc:312] APPLY: finished
1120 03:01:03.348402 (+   210us) log.cc:844] Serialized 29 byte log entry
1120 03:01:03.348562 (+   160us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.348625 (+    63us) write_op.cc:459] Released schema lock
1120 03:01:03.348657 (+    32us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:03.348875 (+   218us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.348899 (+    24us) write_op.cc:459] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":726,"apply.run_cpu_time_us":1476,"apply.run_wall_time_us":1476,"num_ops":2,"prepare.queue_time_us":92,"prepare.run_cpu_time_us":1853,"prepare.run_wall_time_us":1934,"replication_time_us":791934}]]}
I20261120 03:01:03.356715  1181 tablet_bootstrap.cc:654] T 59cc37f7ef9c409493bd89d0d2097339 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:03.358785   500 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:45640 (request call id 185) took 1750 ms. Trace:
I20261120 03:01:03.359099   500 rpcz_store.cc:276] 1120 03:01:01.608093 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:02.516893 (+908800us) service_pool.cc:224] Handling call
1120 03:01:02.524590 (+  7697us) catalog_manager.cc:2145] Acquired catalog manager lock
1120 03:01:02.545727 (+ 21137us) catalog_manager.cc:2191] Created new table and tablet info
1120 03:01:03.358185 (+812458us) catalog_manager.cc:2251] Wrote table and tablets to system table
1120 03:01:03.358460 (+   275us) catalog_manager.cc:2285] Inserted table and tablets into CatalogManager maps
1120 03:01:03.358732 (+   272us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:02.560168 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:02.560677 (+   509us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:02.560702 (+    25us) write_op.cc:440] Acquired schema lock
1120 03:01:02.560714 (+    12us) tablet.cc:710] Decoding operations
1120 03:01:02.560989 (+   275us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:02.561041 (+    52us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:02.561075 (+    34us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:01:02.561280 (+   205us) tablet.cc:758] Row locks acquired
1120 03:01:02.561294 (+    14us) write_op.cc:260] PREPARE: finished
1120 03:01:02.561412 (+   118us) write_op.cc:270] Start()
1120 03:01:02.561497 (+    85us) write_op.cc:276] Timestamp: P: 1795143662561346 usec, L: 0
1120 03:01:02.561511 (+    14us) op_driver.cc:350] REPLICATION: starting
1120 03:01:02.561970 (+   459us) log.cc:844] Serialized 433 byte log entry
1120 03:01:03.346736 (+784766us) op_driver.cc:466] REPLICATION: finished
1120 03:01:03.349115 (+  2379us) write_op.cc:301] APPLY: starting
1120 03:01:03.349163 (+    48us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:03.349261 (+    98us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:03.349345 (+    84us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:03.349785 (+   440us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:03.349911 (+   126us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
1120 03:01:03.349953 (+    42us) write_op.cc:312] APPLY: finished
1120 03:01:03.356252 (+  6299us) log.cc:844] Serialized 29 byte log entry
1120 03:01:03.356409 (+   157us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.356469 (+    60us) write_op.cc:459] Released schema lock
1120 03:01:03.356499 (+    30us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"child_traces":[["op",{"apply.queue_time_us":2262,"num_ops":2,"prepare.queue_time_us":109,"prepare.run_cpu_time_us":1947,"prepare.run_wall_time_us":2050,"replication_time_us":785152,"thread_start_us":8409,"threads_started":1}]]}
I20261120 03:01:03.361979  1947 tablet.cc:1872] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20261120 03:01:03.367137   500 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_9821147009834f8482ae7e45dfd18e3c" } modify_external_catalogs: true
I20261120 03:01:03.367673   500 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_9821147009834f8482ae7e45dfd18e3c" } modify_external_catalogs: true
I20261120 03:01:03.375991  1791 raft_consensus.cc:2966] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
W20261120 03:01:03.377584   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 50396.7us
I20261120 03:01:03.352861  1324 ts_tablet_manager.cc:1480] T 26f8ba69516c447dbd91e0d87cd519d3 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.162s	user 0.004s	sys 0.002s
I20261120 03:01:03.379961  1181 tablet_bootstrap.cc:492] T 59cc37f7ef9c409493bd89d0d2097339 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:03.380240  1324 tablet_bootstrap.cc:492] T bed7910ab7c140199de04e306c532969 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:01:03.380350  1181 ts_tablet_manager.cc:1449] T 59cc37f7ef9c409493bd89d0d2097339 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.039s	user 0.012s	sys 0.002s
I20261120 03:01:03.382741  1181 raft_consensus.cc:366] T 59cc37f7ef9c409493bd89d0d2097339 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.383342  1181 raft_consensus.cc:392] T 59cc37f7ef9c409493bd89d0d2097339 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:03.384975  1761 ts_tablet_manager.cc:1962] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:03.392558   495 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:45640 (request call id 190) took 1409 ms. Trace:
I20261120 03:01:03.393170  1181 raft_consensus.cc:787] T 59cc37f7ef9c409493bd89d0d2097339 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:03.393473   494 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 0db7db25d779459ebde532f938493975
I20261120 03:01:03.393810  1181 consensus_queue.cc:261] T 59cc37f7ef9c409493bd89d0d2097339 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.394829   494 catalog_manager.cc:6952] ReplaceTablet: tablet 0db7db25d779459ebde532f938493975 deleted and replaced by tablet 2936e79caca943258b39849505c59f50
I20261120 03:01:03.392911   495 rpcz_store.cc:276] 1120 03:01:01.983330 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:02.575445 (+592115us) service_pool.cc:224] Handling call
1120 03:01:02.581909 (+  6464us) catalog_manager.cc:2145] Acquired catalog manager lock
1120 03:01:02.607832 (+ 25923us) catalog_manager.cc:2191] Created new table and tablet info
1120 03:01:03.391874 (+784042us) catalog_manager.cc:2251] Wrote table and tablets to system table
1120 03:01:03.392168 (+   294us) catalog_manager.cc:2285] Inserted table and tablets into CatalogManager maps
1120 03:01:03.392496 (+   328us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:02.625880 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:02.626302 (+   422us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:02.626328 (+    26us) write_op.cc:440] Acquired schema lock
1120 03:01:02.626341 (+    13us) tablet.cc:710] Decoding operations
1120 03:01:02.626633 (+   292us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:02.626668 (+    35us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:02.626690 (+    22us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:01:02.626878 (+   188us) tablet.cc:758] Row locks acquired
1120 03:01:02.626890 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:01:02.627006 (+   116us) write_op.cc:270] Start()
1120 03:01:02.627095 (+    89us) write_op.cc:276] Timestamp: P: 1795143662626936 usec, L: 0
1120 03:01:02.627110 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:01:02.627544 (+   434us) log.cc:844] Serialized 433 byte log entry
1120 03:01:03.365990 (+738446us) op_driver.cc:466] REPLICATION: finished
1120 03:01:03.374276 (+  8286us) write_op.cc:301] APPLY: starting
1120 03:01:03.374341 (+    65us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:03.390398 (+ 16057us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:03.390521 (+   123us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:03.391004 (+   483us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:03.391154 (+   150us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
1120 03:01:03.391194 (+    40us) write_op.cc:312] APPLY: finished
1120 03:01:03.391406 (+   212us) log.cc:844] Serialized 29 byte log entry
1120 03:01:03.391582 (+   176us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.391643 (+    61us) write_op.cc:459] Released schema lock
1120 03:01:03.391679 (+    36us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:03.391928 (+   249us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.391951 (+    23us) write_op.cc:459] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":8098,"apply.run_cpu_time_us":1754,"apply.run_wall_time_us":17640,"mutex_wait_us":16761,"num_ops":2,"prepare.queue_time_us":16888,"prepare.run_cpu_time_us":1850,"prepare.run_wall_time_us":1850,"replication_time_us":738798}]]}
I20261120 03:01:03.395149   494 rpcz_store.cc:275] Call kudu.master.MasterService.ReplaceTablet from 127.0.0.1:45640 (request call id 187) took 1675 ms. Trace:
I20261120 03:01:03.395839  1181 ts_tablet_manager.cc:1480] T 59cc37f7ef9c409493bd89d0d2097339 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.015s	user 0.005s	sys 0.000s
I20261120 03:01:03.396631  1181 tablet_bootstrap.cc:492] T 1faba22e8e38418383b86e42e904a761 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:03.395778   495 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:45640:
table { table_name: "default.Table_92f6cc71382f4deba2472c49ae8f6a36" } new_table_name: "default.Table_36247824df9547f8b6c7cb6ca56fb0ea" modify_external_catalogs: true
I20261120 03:01:03.397657   385 master-stress-test.cc:581] Killed test thread 1/10
I20261120 03:01:03.397897   385 master-stress-test.cc:579] Killing test thread 2/10
I20261120 03:01:03.398149   385 master-stress-test.cc:581] Killed test thread 2/10
I20261120 03:01:03.398342   385 master-stress-test.cc:579] Killing test thread 3/10
I20261120 03:01:03.398550   385 master-stress-test.cc:581] Killed test thread 3/10
I20261120 03:01:03.398728   385 master-stress-test.cc:579] Killing test thread 4/10
I20261120 03:01:03.395416   494 rpcz_store.cc:276] 1120 03:01:01.719714 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:02.524483 (+804769us) service_pool.cc:224] Handling call
1120 03:01:03.395084 (+870601us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:02.578594 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:02.579123 (+   529us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:02.579148 (+    25us) write_op.cc:440] Acquired schema lock
1120 03:01:02.579163 (+    15us) tablet.cc:710] Decoding operations
1120 03:01:02.579501 (+   338us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:02.579541 (+    40us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:02.579564 (+    23us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:01:02.579772 (+   208us) tablet.cc:758] Row locks acquired
1120 03:01:02.579786 (+    14us) write_op.cc:260] PREPARE: finished
1120 03:01:02.580046 (+   260us) write_op.cc:270] Start()
1120 03:01:02.580144 (+    98us) write_op.cc:276] Timestamp: P: 1795143662579973 usec, L: 0
1120 03:01:02.580161 (+    17us) op_driver.cc:350] REPLICATION: starting
1120 03:01:02.580655 (+   494us) log.cc:844] Serialized 656 byte log entry
1120 03:01:03.355445 (+774790us) op_driver.cc:466] REPLICATION: finished
1120 03:01:03.356730 (+  1285us) write_op.cc:301] APPLY: starting
1120 03:01:03.356778 (+    48us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:03.356895 (+   117us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:03.356990 (+    95us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:03.357326 (+   336us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:03.357518 (+   192us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:01:03.357573 (+    55us) write_op.cc:312] APPLY: finished
1120 03:01:03.357744 (+   171us) log.cc:844] Serialized 29 byte log entry
1120 03:01:03.357887 (+   143us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.357945 (+    58us) write_op.cc:459] Released schema lock
1120 03:01:03.357975 (+    30us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"child_traces":[["op",{"apply.queue_time_us":1118,"apply.run_cpu_time_us":1499,"apply.run_wall_time_us":42374,"mutex_wait_us":39360,"num_ops":2,"prepare.queue_time_us":39496,"prepare.run_cpu_time_us":2093,"prepare.run_wall_time_us":2345,"replication_time_us":775200}]]}
I20261120 03:01:03.405896   494 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45640:
name: "default.Table_d1dabef467c84c168eaedfda4f61a259"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:01:03.407411   494 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.Table_d1dabef467c84c168eaedfda4f61a259 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:01:03.414951   896 tablet_service.cc:1542] Processing DeleteTablet for tablet 0db7db25d779459ebde532f938493975 with delete_type TABLET_DATA_DELETED (replaced by tablet 2936e79caca943258b39849505c59f50) from {username='slave'} at 127.0.0.1:44896
I20261120 03:01:03.423799  1837 consensus_queue.cc:1049] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:03.424623  1181 tablet_bootstrap.cc:654] T 1faba22e8e38418383b86e42e904a761 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:03.434474   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 88b9e9692a874ae993dde4b9e2586a70 (table default.Table_e4c5ff8f28ae4efca67257942a4fcb77 [id=b30488f6d14b4c9ab1566f2376be602a]) successfully deleted
I20261120 03:01:03.446558  1181 tablet_bootstrap.cc:492] T 1faba22e8e38418383b86e42e904a761 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:03.447121  1181 ts_tablet_manager.cc:1449] T 1faba22e8e38418383b86e42e904a761 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.051s	user 0.013s	sys 0.000s
I20261120 03:01:03.450513  1181 raft_consensus.cc:366] T 1faba22e8e38418383b86e42e904a761 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.451179  1181 raft_consensus.cc:392] T 1faba22e8e38418383b86e42e904a761 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:03.451442  1181 raft_consensus.cc:787] T 1faba22e8e38418383b86e42e904a761 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
W20261120 03:01:03.451776   486 catalog_manager.cc:5048] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 68179488b9bc4e03b65a9f1d434c95b5 because the tablet was not found. No further retry: Not found: Tablet not found: 68179488b9bc4e03b65a9f1d434c95b5
I20261120 03:01:03.452098  1181 consensus_queue.cc:261] T 1faba22e8e38418383b86e42e904a761 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.454015  1181 ts_tablet_manager.cc:1480] T 1faba22e8e38418383b86e42e904a761 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.007s	user 0.003s	sys 0.000s
I20261120 03:01:03.465607  1180 tablet_bootstrap.cc:654] T 73c993944b0a4977b55d40ad63bb69e3 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:03.470731  1045 tablet_service.cc:1542] Processing DeleteTablet for tablet 0db7db25d779459ebde532f938493975 with delete_type TABLET_DATA_DELETED (replaced by tablet 2936e79caca943258b39849505c59f50) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:03.471737   750 tablet_service.cc:1542] Processing DeleteTablet for tablet 0db7db25d779459ebde532f938493975 with delete_type TABLET_DATA_DELETED (replaced by tablet 2936e79caca943258b39849505c59f50) from {username='slave'} at 127.0.0.1:52644
I20261120 03:01:03.479923  1761 ts_tablet_manager.cc:1975] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 2.2
I20261120 03:01:03.480401  1761 log.cc:1199] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/1885397e42774a89bb347bc776284bc0
I20261120 03:01:03.481269  1761 ts_tablet_manager.cc:1996] T 1885397e42774a89bb347bc776284bc0 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:01:03.483668  1181 tablet_bootstrap.cc:492] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:03.489480  1181 tablet_bootstrap.cc:654] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:03.492735  1761 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:52644 (request call id 136) took 3613 ms. Trace:
I20261120 03:01:03.493124  1761 rpcz_store.cc:276] 1120 03:00:59.879742 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.884312 (+  4570us) service_pool.cc:224] Handling call
1120 03:01:03.343881 (+3459569us) ts_tablet_manager.cc:1197] Deleting tablet 1885397e42774a89bb347bc776284bc0
1120 03:01:03.343933 (+    52us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:03.430745 (+ 86812us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:03.479788 (+ 49043us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:03.492675 (+ 12887us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3459091}
I20261120 03:01:03.495096  1761 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:52644 (request call id 141) took 3565 ms. Trace:
I20261120 03:01:03.506418  1761 rpcz_store.cc:276] 1120 03:00:59.929877 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.929976 (+    99us) service_pool.cc:224] Handling call
1120 03:01:03.494727 (+3564751us) ts_tablet_manager.cc:1197] Deleting tablet c6a65c1a4a574961bec657ad7c9048e0
1120 03:01:03.494796 (+    69us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:03.495035 (+   239us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3564145}
I20261120 03:01:03.508038  1761 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:52644 (request call id 140) took 3578 ms. Trace:
I20261120 03:01:03.508431  1761 rpcz_store.cc:276] 1120 03:00:59.929561 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:00:59.930725 (+  1164us) service_pool.cc:224] Handling call
1120 03:01:03.507747 (+3577022us) ts_tablet_manager.cc:1197] Deleting tablet 014bd9c83be34c31b90fe4bcf31f1649
1120 03:01:03.507798 (+    51us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:03.507985 (+   187us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3576507}
I20261120 03:01:03.513094  1915 tablet_replica.cc:333] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:01:03.514030  1915 raft_consensus.cc:2440] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:03.514603  1915 raft_consensus.cc:2469] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:03.515362  1181 tablet_bootstrap.cc:492] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:03.515861  1181 ts_tablet_manager.cc:1449] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.032s	user 0.009s	sys 0.005s
I20261120 03:01:03.516479  1915 ts_tablet_manager.cc:1962] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:03.517360  1761 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:52644 (request call id 142) took 3238 ms. Trace:
I20261120 03:01:03.518352  1931 raft_consensus.cc:534] T 26f8ba69516c447dbd91e0d87cd519d3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:03.518230  1181 raft_consensus.cc:366] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.518903  1181 raft_consensus.cc:392] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:03.518764  1931 raft_consensus.cc:556] T 26f8ba69516c447dbd91e0d87cd519d3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.519140  1181 raft_consensus.cc:787] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:03.519907  1181 consensus_queue.cc:261] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.521163  1931 leader_election.cc:290] T 26f8ba69516c447dbd91e0d87cd519d3 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:03.521962  1181 ts_tablet_manager.cc:1480] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.006s	user 0.004s	sys 0.000s
I20261120 03:01:03.522809  1181 tablet_bootstrap.cc:492] T 0179a59749a14168a840e9d4e610be1f P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:03.531143  1324 tablet_bootstrap.cc:654] T bed7910ab7c140199de04e306c532969 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:03.536631  1181 tablet_bootstrap.cc:654] T 0179a59749a14168a840e9d4e610be1f P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:03.517654  1761 rpcz_store.cc:276] 1120 03:01:00.278500 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:00.288211 (+  9711us) service_pool.cc:224] Handling call
1120 03:01:03.516956 (+3228745us) ts_tablet_manager.cc:1197] Deleting tablet b549a06d15db449b851fe9549a963416
1120 03:01:03.517013 (+    57us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:03.517309 (+   296us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3228142}
I20261120 03:01:03.541721   496 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
I20261120 03:01:03.542629   496 catalog_manager.cc:5856] T a04da9cb8f004443aa380902ea5427c5 P 823abdbd90ba47bbacad0f8dcff78234 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65). New cstate: current_term: 1 leader_uuid: "5beeefe8bef54caeac463d05d8c30fa6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } }
I20261120 03:01:03.548296  1045 tablet_service.cc:1495] Processing CreateTablet for tablet 5c3cfc12ab6041ce80b852177f2b6854 (DEFAULT_TABLE table=default.Table_9821147009834f8482ae7e45dfd18e3c [id=967887ed7af440ea88625626876fa758]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.549347  1045 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5c3cfc12ab6041ce80b852177f2b6854. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.549127   750 tablet_service.cc:1495] Processing CreateTablet for tablet 5c3cfc12ab6041ce80b852177f2b6854 (DEFAULT_TABLE table=default.Table_9821147009834f8482ae7e45dfd18e3c [id=967887ed7af440ea88625626876fa758]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.550380   756 tablet_service.cc:1495] Processing CreateTablet for tablet 8044a04a74974508af72bcbbfd574629 (DEFAULT_TABLE table=default.Table_5e1840985084419290ec475c0971cfe4 [id=a0d872c3f9644aed8e1fb3b3359a5fc9]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.552408   751 tablet_service.cc:1495] Processing CreateTablet for tablet ac349a09a0524b2d87289c3dd53adf6a (DEFAULT_TABLE table=default.Table_e4c5ff8f28ae4efca67257942a4fcb77 [id=b30488f6d14b4c9ab1566f2376be602a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.553495  1059 tablet_service.cc:1495] Processing CreateTablet for tablet ac349a09a0524b2d87289c3dd53adf6a (DEFAULT_TABLE table=default.Table_e4c5ff8f28ae4efca67257942a4fcb77 [id=b30488f6d14b4c9ab1566f2376be602a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.553678  1051 tablet_service.cc:1495] Processing CreateTablet for tablet 86d3a4c569f9485ebad6432ff95e4174 (DEFAULT_TABLE table=default.Table_40975331cb484f45b9592c00eab3cc54 [id=2c68613c88184e2989f91b1521bb3317]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.553681  1046 tablet_service.cc:1495] Processing CreateTablet for tablet 8044a04a74974508af72bcbbfd574629 (DEFAULT_TABLE table=default.Table_5e1840985084419290ec475c0971cfe4 [id=a0d872c3f9644aed8e1fb3b3359a5fc9]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.554605  1059 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ac349a09a0524b2d87289c3dd53adf6a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.554476   752 tablet_service.cc:1495] Processing CreateTablet for tablet 86d3a4c569f9485ebad6432ff95e4174 (DEFAULT_TABLE table=default.Table_40975331cb484f45b9592c00eab3cc54 [id=2c68613c88184e2989f91b1521bb3317]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.556991  1055 tablet_service.cc:1495] Processing CreateTablet for tablet 1f3f49b17ac44aa7a8e9e803fa0c78f7 (DEFAULT_TABLE table=default.Table_5cbde2434cf04febac447e8bb2c628d5 [id=a4b20878e3b04261b3826cc6edbd3196]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.557813  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "26f8ba69516c447dbd91e0d87cd519d3" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:03.559046  1046 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8044a04a74974508af72bcbbfd574629. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.560082  1055 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1f3f49b17ac44aa7a8e9e803fa0c78f7. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.561021  1051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 86d3a4c569f9485ebad6432ff95e4174. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.564581  1761 tablet_replica.cc:333] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:01:03.565367  1761 raft_consensus.cc:2440] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:03.565822  1761 raft_consensus.cc:2469] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:03.567665  1761 ts_tablet_manager.cc:1962] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:03.571373  1181 tablet_bootstrap.cc:492] T 0179a59749a14168a840e9d4e610be1f P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:03.571552   756 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8044a04a74974508af72bcbbfd574629. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.571798  1181 ts_tablet_manager.cc:1449] T 0179a59749a14168a840e9d4e610be1f P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.049s	user 0.006s	sys 0.003s
I20261120 03:01:03.574199  1181 raft_consensus.cc:366] T 0179a59749a14168a840e9d4e610be1f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.574680   751 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ac349a09a0524b2d87289c3dd53adf6a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.574800  1181 raft_consensus.cc:392] T 0179a59749a14168a840e9d4e610be1f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:03.575053  1181 raft_consensus.cc:787] T 0179a59749a14168a840e9d4e610be1f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:03.578346  1181 consensus_queue.cc:261] T 0179a59749a14168a840e9d4e610be1f P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.560261   486 catalog_manager.cc:5081] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): tablet 1885397e42774a89bb347bc776284bc0 (table default.Table_61cb1e03b7e442a8822e32a2295866f3 [id=b5b565c2c2224675baada2ae9b42be54]) successfully deleted
W20261120 03:01:03.579392   486 catalog_manager.cc:5048] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet c6a65c1a4a574961bec657ad7c9048e0 because the tablet was not found. No further retry: Not found: Tablet not found: c6a65c1a4a574961bec657ad7c9048e0
I20261120 03:01:03.578586   752 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 86d3a4c569f9485ebad6432ff95e4174. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:01:03.580018   486 catalog_manager.cc:5048] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet 014bd9c83be34c31b90fe4bcf31f1649 because the tablet was not found. No further retry: Not found: Tablet not found: 014bd9c83be34c31b90fe4bcf31f1649
I20261120 03:01:03.580384  1181 ts_tablet_manager.cc:1480] T 0179a59749a14168a840e9d4e610be1f P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.008s	user 0.006s	sys 0.001s
I20261120 03:01:03.581218  1181 tablet_bootstrap.cc:492] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
W20261120 03:01:03.594156   486 catalog_manager.cc:5048] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): delete failed for tablet b549a06d15db449b851fe9549a963416 because the tablet was not found. No further retry: Not found: Tablet not found: b549a06d15db449b851fe9549a963416
I20261120 03:01:03.595638   750 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5c3cfc12ab6041ce80b852177f2b6854. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.596446   753 tablet_service.cc:1495] Processing CreateTablet for tablet 1f3f49b17ac44aa7a8e9e803fa0c78f7 (DEFAULT_TABLE table=default.Table_5cbde2434cf04febac447e8bb2c628d5 [id=a4b20878e3b04261b3826cc6edbd3196]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.596993  1181 tablet_bootstrap.cc:654] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:03.597489   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1f3f49b17ac44aa7a8e9e803fa0c78f7. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.612412  1181 tablet_bootstrap.cc:492] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:03.612831  1181 ts_tablet_manager.cc:1449] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.032s	user 0.008s	sys 0.000s
I20261120 03:01:03.615271  1181 raft_consensus.cc:366] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.616003  1181 raft_consensus.cc:392] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:03.616259  1181 raft_consensus.cc:787] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:03.617053  1181 consensus_queue.cc:261] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
W20261120 03:01:03.621685   495 catalog_manager.cc:6124] Failed to send alter table request: Not found: failed to reset TS proxy: No leader found: Tablet 8935b23c661c4c1ca7bde7c893927502 consensus state has no leader
I20261120 03:01:03.625697  1181 ts_tablet_manager.cc:1480] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.013s	user 0.006s	sys 0.000s
I20261120 03:01:03.626835  1181 tablet_bootstrap.cc:492] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:03.627561   385 master-stress-test.cc:581] Killed test thread 4/10
I20261120 03:01:03.627801   385 master-stress-test.cc:579] Killing test thread 5/10
I20261120 03:01:03.628793   495 master_service.cc:965] ReplaceTablet: received request to replace tablet 1faba22e8e38418383b86e42e904a761 from {username='slave'} at 127.0.0.1:45640
W20261120 03:01:03.631197   709 leader_election.cc:343] T 26f8ba69516c447dbd91e0d87cd519d3 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:03.632439  1181 tablet_bootstrap.cc:654] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:03.646858  1059 tablet_service.cc:1495] Processing CreateTablet for tablet 81d7cb5526944095b7054464971e8fa4 (DEFAULT_TABLE table=default.Table_c1e41dc463834422869aec7e3bace0f1 [id=0d5e41d23488417f94be9626cc967dbc]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.647902  1059 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 81d7cb5526944095b7054464971e8fa4. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.648305  1181 tablet_bootstrap.cc:492] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:03.648679  1181 ts_tablet_manager.cc:1449] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.022s	user 0.004s	sys 0.004s
I20261120 03:01:03.651050  1181 raft_consensus.cc:366] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.651657  1181 raft_consensus.cc:392] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:03.651942  1181 raft_consensus.cc:787] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:03.652541  1181 consensus_queue.cc:261] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.654511  1181 ts_tablet_manager.cc:1480] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.006s	user 0.007s	sys 0.000s
I20261120 03:01:03.662681  1181 tablet_bootstrap.cc:492] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:03.668267  1181 tablet_bootstrap.cc:654] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:03.685247   757 tablet_service.cc:1495] Processing CreateTablet for tablet 81d7cb5526944095b7054464971e8fa4 (DEFAULT_TABLE table=default.Table_c1e41dc463834422869aec7e3bace0f1 [id=0d5e41d23488417f94be9626cc967dbc]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.686347   757 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 81d7cb5526944095b7054464971e8fa4. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.716518  1931 raft_consensus.cc:534] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:03.717036  1931 raft_consensus.cc:556] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.719483  1931 leader_election.cc:290] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:03.722394  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "73c993944b0a4977b55d40ad63bb69e3" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:03.728121  1761 ts_tablet_manager.cc:1975] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:03.730767  1181 tablet_bootstrap.cc:492] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:03.731227  1181 ts_tablet_manager.cc:1449] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.069s	user 0.015s	sys 0.008s
I20261120 03:01:03.732652  1893 raft_consensus.cc:534] T 9dadc05b726346309d6b350f9abb4abb P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:03.733109  1893 raft_consensus.cc:556] T 9dadc05b726346309d6b350f9abb4abb P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:03.670733   857 leader_election.cc:304] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:01:03.680371   897 tablet_service.cc:1495] Processing CreateTablet for tablet 81d7cb5526944095b7054464971e8fa4 (DEFAULT_TABLE table=default.Table_c1e41dc463834422869aec7e3bace0f1 [id=0d5e41d23488417f94be9626cc967dbc]), partition=RANGE (key) PARTITION UNBOUNDED
W20261120 03:01:03.734551   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 310974us
I20261120 03:01:03.735139  1893 leader_election.cc:290] T 9dadc05b726346309d6b350f9abb4abb P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:01:03.684437   902 tablet_service.cc:1495] Processing CreateTablet for tablet 1f3f49b17ac44aa7a8e9e803fa0c78f7 (DEFAULT_TABLE table=default.Table_5cbde2434cf04febac447e8bb2c628d5 [id=a4b20878e3b04261b3826cc6edbd3196]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.685652   903 tablet_service.cc:1495] Processing CreateTablet for tablet 990f99e925ca4701bbc664af1c8180e5 (DEFAULT_TABLE table=default.Table_643328a529cb4231a409460bdae21438 [id=f96415c7bb8542d8b2a0d277a58215b8]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.686710   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "26f8ba69516c447dbd91e0d87cd519d3" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:03.692334   896 tablet_service.cc:1495] Processing CreateTablet for tablet 5c3cfc12ab6041ce80b852177f2b6854 (DEFAULT_TABLE table=default.Table_9821147009834f8482ae7e45dfd18e3c [id=967887ed7af440ea88625626876fa758]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.693614   895 tablet_service.cc:1495] Processing CreateTablet for tablet ac349a09a0524b2d87289c3dd53adf6a (DEFAULT_TABLE table=default.Table_e4c5ff8f28ae4efca67257942a4fcb77 [id=b30488f6d14b4c9ab1566f2376be602a]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.681991   898 tablet_service.cc:1495] Processing CreateTablet for tablet 8044a04a74974508af72bcbbfd574629 (DEFAULT_TABLE table=default.Table_5e1840985084419290ec475c0971cfe4 [id=a0d872c3f9644aed8e1fb3b3359a5fc9]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.720849   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "73c993944b0a4977b55d40ad63bb69e3" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:03.739251   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9dadc05b726346309d6b350f9abb4abb" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:03.735319   897 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 81d7cb5526944095b7054464971e8fa4. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.683053   900 tablet_service.cc:1495] Processing CreateTablet for tablet 86d3a4c569f9485ebad6432ff95e4174 (DEFAULT_TABLE table=default.Table_40975331cb484f45b9592c00eab3cc54 [id=2c68613c88184e2989f91b1521bb3317]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.740996   903 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 990f99e925ca4701bbc664af1c8180e5. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.740823  1181 raft_consensus.cc:366] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.741438  1181 raft_consensus.cc:392] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:03.741691  1181 raft_consensus.cc:787] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:03.742012   900 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 86d3a4c569f9485ebad6432ff95e4174. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.742326  1181 consensus_queue.cc:261] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.743052   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1f3f49b17ac44aa7a8e9e803fa0c78f7. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.746260   898 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 8044a04a74974508af72bcbbfd574629. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.750428   895 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet ac349a09a0524b2d87289c3dd53adf6a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.754637   896 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 5c3cfc12ab6041ce80b852177f2b6854. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.754993  1181 ts_tablet_manager.cc:1480] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.024s	user 0.006s	sys 0.000s
I20261120 03:01:03.757920  1181 tablet_bootstrap.cc:492] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:03.759174  1893 raft_consensus.cc:534] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:03.759622  1893 raft_consensus.cc:556] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:03.761691  1893 leader_election.cc:290] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
I20261120 03:01:03.765563   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "600a18b7e5104a78935d906b8adfde6f" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:03.766567   919 raft_consensus.cc:2628] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 823abdbd90ba47bbacad0f8dcff78234 in current term 1: Already voted for candidate 3d30c2428730481db9b0ffd279dd3440 in this term.
I20261120 03:01:03.776594  1181 tablet_bootstrap.cc:654] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:03.785458  1791 raft_consensus.cc:534] T ad49c8e00aca4a88a091baceae320477 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:03.785985  1791 raft_consensus.cc:556] T ad49c8e00aca4a88a091baceae320477 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.788780  1181 tablet_bootstrap.cc:492] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:03.789166  1181 ts_tablet_manager.cc:1449] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.031s	user 0.007s	sys 0.001s
I20261120 03:01:03.789314  1791 leader_election.cc:290] T ad49c8e00aca4a88a091baceae320477 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:03.793795  1181 raft_consensus.cc:366] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.794497  1181 raft_consensus.cc:392] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:03.794570   500 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 5c3cfc12ab6041ce80b852177f2b6854
I20261120 03:01:03.794768  1181 raft_consensus.cc:787] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:03.789918  1791 raft_consensus.cc:3024] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:03.796486  1791 raft_consensus.cc:534] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:03.796684   500 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:45640 (request call id 192) took 1067 ms. Trace:
I20261120 03:01:03.796878  1791 raft_consensus.cc:3296] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:03.797833  1791 raft_consensus.cc:556] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.797024   500 rpcz_store.cc:276] 1120 03:01:02.729567 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:03.361251 (+631684us) service_pool.cc:224] Handling call
1120 03:01:03.367045 (+  5794us) spinlock_profiling.cc:245] Waited 4.94 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07f2b6 00007f155abfbcf4 00007f155abfbc82 00007f155abfbbb1 00007f155abfbb2c 00007f155abfade2 00007f1558e990bc 00007f1558e984f6
1120 03:01:03.368082 (+  1037us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:01:03.368573 (+   491us) catalog_manager.cc:2859] Modifying in-memory table state
1120 03:01:03.368685 (+   112us) catalog_manager.cc:2868] Locking tablets
1120 03:01:03.502861 (+134176us) catalog_manager.cc:2882] Removing table and tablets from system table
1120 03:01:03.691936 (+189075us) catalog_manager.cc:2900] Removing table from by-name map
1120 03:01:03.794382 (+102446us) catalog_manager.cc:2916] Committing in-memory state
1120 03:01:03.794474 (+    92us) catalog_manager.cc:2920] Aborting table tasks
1120 03:01:03.796614 (+  2140us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:03.504770 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:03.505137 (+   367us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:03.505161 (+    24us) write_op.cc:440] Acquired schema lock
1120 03:01:03.505174 (+    13us) tablet.cc:710] Decoding operations
1120 03:01:03.505466 (+   292us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:03.505502 (+    36us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:03.505523 (+    21us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:01:03.505792 (+   269us) tablet.cc:758] Row locks acquired
1120 03:01:03.505805 (+    13us) write_op.cc:260] PREPARE: finished
1120 03:01:03.505915 (+   110us) write_op.cc:270] Start()
1120 03:01:03.505994 (+    79us) write_op.cc:276] Timestamp: P: 1795143663505855 usec, L: 0
1120 03:01:03.506009 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:01:03.506560 (+   551us) log.cc:844] Serialized 725 byte log entry
1120 03:01:03.686694 (+180134us) op_driver.cc:466] REPLICATION: finished
1120 03:01:03.687425 (+   731us) write_op.cc:301] APPLY: starting
1120 03:01:03.687490 (+    65us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:03.687607 (+   117us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:03.687708 (+   101us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:03.687947 (+   239us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:03.688082 (+   135us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
1120 03:01:03.688124 (+    42us) write_op.cc:312] APPLY: finished
1120 03:01:03.688329 (+   205us) log.cc:844] Serialized 29 byte log entry
1120 03:01:03.688492 (+   163us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.688554 (+    62us) write_op.cc:459] Released schema lock
1120 03:01:03.688589 (+    35us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:03.688811 (+   222us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.688834 (+    23us) write_op.cc:459] Released schema lock
Metrics: {"spinlock_wait_cycles":11370880,"child_traces":[["op",{"apply.queue_time_us":589,"apply.run_cpu_time_us":1353,"apply.run_wall_time_us":1352,"num_ops":2,"prepare.queue_time_us":117,"prepare.run_cpu_time_us":1977,"prepare.run_wall_time_us":1998,"replication_time_us":180580}]]}
I20261120 03:01:03.798933  1181 consensus_queue.cc:261] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
W20261120 03:01:03.799911   489 catalog_manager.cc:4863] Async tablet task AlterTable RPC for tablet 8935b23c661c4c1ca7bde7c893927502 (table default.Table_36247824df9547f8b6c7cb6ca56fb0ea [id=51df6e371d3441578e57ea5597ee81a6], current schema version=2) failed: Not found: failed to reset TS proxy: No leader found: Tablet 8935b23c661c4c1ca7bde7c893927502 consensus state has no leader
I20261120 03:01:03.803365  1181 ts_tablet_manager.cc:1480] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.014s	user 0.007s	sys 0.000s
I20261120 03:01:03.804275  1181 tablet_bootstrap.cc:492] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:03.806150   495 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet 1faba22e8e38418383b86e42e904a761
I20261120 03:01:03.806285  1761 log.cc:1199] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/a1192a7f050a4f5387a30b2216dec5f0
I20261120 03:01:03.806980  1761 ts_tablet_manager.cc:1996] T a1192a7f050a4f5387a30b2216dec5f0 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting consensus metadata
I20261120 03:01:03.808116   495 catalog_manager.cc:6952] ReplaceTablet: tablet 1faba22e8e38418383b86e42e904a761 deleted and replaced by tablet 4ebd4502672049d9ae09d4446e1a3447
I20261120 03:01:03.809000  1761 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:52644 (request call id 143) took 3492 ms. Trace:
I20261120 03:01:03.809311  1761 rpcz_store.cc:276] 1120 03:01:00.316712 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:00.317222 (+   510us) service_pool.cc:224] Handling call
1120 03:01:03.538988 (+3221766us) ts_tablet_manager.cc:1197] Deleting tablet a1192a7f050a4f5387a30b2216dec5f0
1120 03:01:03.564447 (+ 25459us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:03.661534 (+ 97087us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:03.728000 (+ 66466us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:03.808943 (+ 80943us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":3221250}
I20261120 03:01:03.809962   901 tablet_service.cc:1542] Processing DeleteTablet for tablet 5c3cfc12ab6041ce80b852177f2b6854 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:01:03 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:01:03.811218   901 tablet_service.cc:1542] Processing DeleteTablet for tablet 1faba22e8e38418383b86e42e904a761 with delete_type TABLET_DATA_DELETED (replaced by tablet 4ebd4502672049d9ae09d4446e1a3447) from {username='slave'} at 127.0.0.1:44896
I20261120 03:01:03.811415  1324 tablet_bootstrap.cc:492] T bed7910ab7c140199de04e306c532969 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:01:03.811805  1324 ts_tablet_manager.cc:1449] T bed7910ab7c140199de04e306c532969 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.432s	user 0.006s	sys 0.003s
I20261120 03:01:03.816610  1918 ts_tablet_manager.cc:1975] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:03.817028   499 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
I20261120 03:01:03.817354  1918 log.cc:1199] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/a1192a7f050a4f5387a30b2216dec5f0
W20261120 03:01:03.728958   709 leader_election.cc:343] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:03.736896   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9dadc05b726346309d6b350f9abb4abb" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:03.818958  1181 tablet_bootstrap.cc:654] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:03.819109   499 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 73c993944b0a4977b55d40ad63bb69e3 (table default.Table_a96b0ac352c545da9b955502704e266b [id=1cdd4bbccdbf4449ba67b9aa7089a6fd]). Expected version 1 got 0
I20261120 03:01:03.819701  1011 leader_election.cc:304] T 9dadc05b726346309d6b350f9abb4abb P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:01:03.820681  1893 raft_consensus.cc:2966] T 9dadc05b726346309d6b350f9abb4abb P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20261120 03:01:03.758693   710 leader_election.cc:343] T 26f8ba69516c447dbd91e0d87cd519d3 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:03.764647   773 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "600a18b7e5104a78935d906b8adfde6f" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:03.820817   499 catalog_manager.cc:5856] T 9dadc05b726346309d6b350f9abb4abb P 5beeefe8bef54caeac463d05d8c30fa6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66). New cstate: current_term: 1 leader_uuid: "3d30c2428730481db9b0ffd279dd3440" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } }
I20261120 03:01:03.821558   710 leader_election.cc:304] T 26f8ba69516c447dbd91e0d87cd519d3 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:03.821856   773 raft_consensus.cc:2628] T 600a18b7e5104a78935d906b8adfde6f P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 823abdbd90ba47bbacad0f8dcff78234 in current term 1: Already voted for candidate 5beeefe8bef54caeac463d05d8c30fa6 in this term.
W20261120 03:01:03.822446   710 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 63794.6us
I20261120 03:01:03.823141  1011 leader_election.cc:304] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:01:03.823849  1893 raft_consensus.cc:3296] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:03.784739   753 tablet_service.cc:1495] Processing CreateTablet for tablet 990f99e925ca4701bbc664af1c8180e5 (DEFAULT_TABLE table=default.Table_643328a529cb4231a409460bdae21438 [id=f96415c7bb8542d8b2a0d277a58215b8]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.828481   753 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 990f99e925ca4701bbc664af1c8180e5. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.840528  1808 raft_consensus.cc:2966] T 26f8ba69516c447dbd91e0d87cd519d3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:03.842585  1181 tablet_bootstrap.cc:492] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:03.843046  1181 ts_tablet_manager.cc:1449] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.039s	user 0.009s	sys 0.007s
I20261120 03:01:03.845902  1181 raft_consensus.cc:366] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.846544  1181 raft_consensus.cc:392] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:03.846814  1181 raft_consensus.cc:787] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:03.847425  1181 consensus_queue.cc:261] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.849982  1059 tablet_service.cc:1542] Processing DeleteTablet for tablet 5c3cfc12ab6041ce80b852177f2b6854 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:01:03 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:03.850886  1324 raft_consensus.cc:366] T bed7910ab7c140199de04e306c532969 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.851529  1324 raft_consensus.cc:392] T bed7910ab7c140199de04e306c532969 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:03.851809  1324 raft_consensus.cc:787] T bed7910ab7c140199de04e306c532969 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:01:03.852383  1324 consensus_queue.cc:261] T bed7910ab7c140199de04e306c532969 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.852758   751 tablet_service.cc:1542] Processing DeleteTablet for tablet 5c3cfc12ab6041ce80b852177f2b6854 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:01:03 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:01:03.854784  1324 ts_tablet_manager.cc:1480] T bed7910ab7c140199de04e306c532969 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.043s	user 0.005s	sys 0.002s
I20261120 03:01:03.860836  1059 tablet_service.cc:1542] Processing DeleteTablet for tablet 1faba22e8e38418383b86e42e904a761 with delete_type TABLET_DATA_DELETED (replaced by tablet 4ebd4502672049d9ae09d4446e1a3447) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:03.863183   756 tablet_service.cc:1542] Processing DeleteTablet for tablet 1faba22e8e38418383b86e42e904a761 with delete_type TABLET_DATA_DELETED (replaced by tablet 4ebd4502672049d9ae09d4446e1a3447) from {username='slave'} at 127.0.0.1:52644
I20261120 03:01:03.875610  1791 leader_election.cc:290] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:03.876255  1918 ts_tablet_manager.cc:1996] T a1192a7f050a4f5387a30b2216dec5f0 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:01:03.877565  1893 raft_consensus.cc:2966] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:03.879186  1918 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 155) took 1295 ms. Trace:
W20261120 03:01:03.879338   710 leader_election.cc:343] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:03.881047   710 leader_election.cc:304] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6; no voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:03.881834  1808 raft_consensus.cc:2966] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20261120 03:01:03.882455   710 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 59089.5us
W20261120 03:01:03.884373   709 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 155435us
W20261120 03:01:03.888059   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 129543us
I20261120 03:01:03.892346  1180 tablet_bootstrap.cc:492] T 73c993944b0a4977b55d40ad63bb69e3 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:01:03.892839  1180 ts_tablet_manager.cc:1449] T 73c993944b0a4977b55d40ad63bb69e3 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.583s	user 0.013s	sys 0.000s
I20261120 03:01:03.879588  1918 rpcz_store.cc:276] 1120 03:01:02.583731 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:02.588271 (+  4540us) service_pool.cc:224] Handling call
1120 03:01:02.927297 (+339026us) ts_tablet_manager.cc:1197] Deleting tablet a1192a7f050a4f5387a30b2216dec5f0
1120 03:01:02.927347 (+    50us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:03.197957 (+270610us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:03.429970 (+232013us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:03.879091 (+449121us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":337640,"thread_start_us":340489,"threads_started":1}
I20261120 03:01:03.900365   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet a1192a7f050a4f5387a30b2216dec5f0 (table default.Table_646d629dfeea413a86bc3edf68c7d6ae [id=aa0b779e8ea84d928af40806a7913f1a]) successfully deleted
I20261120 03:01:03.901590  1918 tablet_replica.cc:333] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:01:03.893244   499 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet daae644853734cd5818a46ecaa5ce7df (table default.Table_2a81e9814fe34f9bb1902e71fd82fd01 [id=e540afdca1234f30bf1950448be502df]). Expected version 1 got 0
I20261120 03:01:03.909912  1918 raft_consensus.cc:2440] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:03.910629  1918 raft_consensus.cc:2469] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:03.910419   499 catalog_manager.cc:5856] T 59cc37f7ef9c409493bd89d0d2097339 P 5beeefe8bef54caeac463d05d8c30fa6 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66). New cstate: current_term: 1 leader_uuid: "3d30c2428730481db9b0ffd279dd3440" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } }
I20261120 03:01:03.912747   499 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 0179a59749a14168a840e9d4e610be1f (table default.Table_cc56bf0ad8344f1a94d5da6e4d77e8c0 [id=b9034a083c7e4133a746b336335d3e42]). Expected version 1 got 0
I20261120 03:01:03.913362   499 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 5da3d2724dd34aed88e6b08983188004 (table default.Table_d2bd3060da1545548f422436c9c7c22e [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]). Expected version 2 got 0
I20261120 03:01:03.914601  1918 ts_tablet_manager.cc:1962] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
W20261120 03:01:03.915175  1012 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 45291.8us
I20261120 03:01:03.916189  1974 raft_consensus.cc:534] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:03.916676  1974 raft_consensus.cc:556] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.922698  1974 leader_election.cc:290] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:03.930449  1181 ts_tablet_manager.cc:1480] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.087s	user 0.028s	sys 0.043s
I20261120 03:01:03.930481   773 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ba85dab0d238487181ce5c1592a2c7fd" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:03.931255   773 raft_consensus.cc:2062] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:01:03.932246  1011 leader_election.cc:304] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:01:03.933403  1324 tablet_bootstrap.cc:492] T 0dca5a2820ad4197b83a804b0310e57c P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:01:03.935480  1761 tablet_replica.cc:333] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6: stopping tablet replica
I20261120 03:01:03.933341   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ba85dab0d238487181ce5c1592a2c7fd" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:03.936203   919 raft_consensus.cc:2062] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:01:03.938246  1761 raft_consensus.cc:2440] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:03.938742  1761 raft_consensus.cc:2469] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:03.941576  1761 ts_tablet_manager.cc:1962] T 9e42a416be884341994d2d5395286798 P 5beeefe8bef54caeac463d05d8c30fa6: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:03.943619  1181 tablet_bootstrap.cc:492] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:03.943696  1972 raft_consensus.cc:3024] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:03.944243  1972 raft_consensus.cc:534] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:03.944617  1972 raft_consensus.cc:3296] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:03.944949  1324 tablet_bootstrap.cc:654] T 0dca5a2820ad4197b83a804b0310e57c P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:03.945582  1973 raft_consensus.cc:534] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:03.945554  1972 raft_consensus.cc:556] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.946107  1973 raft_consensus.cc:556] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.948081  1973 leader_election.cc:290] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:03.949244   773 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "73c993944b0a4977b55d40ad63bb69e3" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:03.949961   773 raft_consensus.cc:2062] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:01:03.950906   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "73c993944b0a4977b55d40ad63bb69e3" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:03.952802  1055 tablet_service.cc:1495] Processing CreateTablet for tablet 990f99e925ca4701bbc664af1c8180e5 (DEFAULT_TABLE table=default.Table_643328a529cb4231a409460bdae21438 [id=f96415c7bb8542d8b2a0d277a58215b8]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:03.953857  1055 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 990f99e925ca4701bbc664af1c8180e5. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:03.955333  1011 leader_election.cc:304] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:01:03.958770  1975 raft_consensus.cc:3024] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:03.959192  1975 raft_consensus.cc:534] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:03.959592  1975 raft_consensus.cc:3296] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:03.960387  1975 raft_consensus.cc:556] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.975198   493 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
I20261120 03:01:03.984408  1915 ts_tablet_manager.cc:1975] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20261120 03:01:03.984912  1915 log.cc:1199] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/0db7db25d779459ebde532f938493975
I20261120 03:01:03.985759  1915 ts_tablet_manager.cc:1996] T 0db7db25d779459ebde532f938493975 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:01:03.990736  1977 raft_consensus.cc:534] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:03.991307  1977 raft_consensus.cc:556] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.995004  1181 tablet_bootstrap.cc:654] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:03.996273  1973 raft_consensus.cc:534] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:03.996699  1973 raft_consensus.cc:556] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.998219  1324 tablet_bootstrap.cc:492] T 0dca5a2820ad4197b83a804b0310e57c P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:01:03.998521  1973 leader_election.cc:290] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:03.999610   773 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ad49c8e00aca4a88a091baceae320477" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:04.002038   773 raft_consensus.cc:2062] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:01:04.001323   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00f7c7d95bae4d55be6b437701989ba3" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:04.002969   777 raft_consensus.cc:2062] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
I20261120 03:01:04.004281  1011 leader_election.cc:304] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:01:04.005079  1011 leader_election.cc:304] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:01:04.008773  1973 raft_consensus.cc:3024] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:04.009603  1973 raft_consensus.cc:534] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:04.010368  1973 raft_consensus.cc:3296] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:04.011009  1324 ts_tablet_manager.cc:1449] T 0dca5a2820ad4197b83a804b0310e57c P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.078s	user 0.010s	sys 0.000s
I20261120 03:01:04.011296  1977 leader_election.cc:290] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:04.013391  1977 raft_consensus.cc:3024] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:03.934852   857 leader_election.cc:304] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6; no voters: 
I20261120 03:01:04.015901  1967 raft_consensus.cc:3024] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:04.016228  1967 raft_consensus.cc:534] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:04.016549  1967 raft_consensus.cc:3296] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:04.016059  1977 raft_consensus.cc:534] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:04.017138  1977 raft_consensus.cc:3296] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:04.018494  1918 ts_tablet_manager.cc:1975] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:04.018586  1977 raft_consensus.cc:556] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.019200  1918 log.cc:1199] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/b549a06d15db449b851fe9549a963416
I20261120 03:01:04.020898  1918 ts_tablet_manager.cc:1996] T b549a06d15db449b851fe9549a963416 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:01:04.021085  1324 raft_consensus.cc:366] T 0dca5a2820ad4197b83a804b0310e57c P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.021705  1324 raft_consensus.cc:392] T 0dca5a2820ad4197b83a804b0310e57c P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:04.021947  1324 raft_consensus.cc:787] T 0dca5a2820ad4197b83a804b0310e57c P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:01:04.022233  1967 raft_consensus.cc:556] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.022540  1324 consensus_queue.cc:261] T 0dca5a2820ad4197b83a804b0310e57c P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.024384  1324 ts_tablet_manager.cc:1480] T 0dca5a2820ad4197b83a804b0310e57c P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.011s	user 0.003s	sys 0.000s
I20261120 03:01:04.025199  1324 tablet_bootstrap.cc:492] T 8935b23c661c4c1ca7bde7c893927502 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:01:04.026412  1967 leader_election.cc:290] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
W20261120 03:01:04.027318   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 92565.6us
I20261120 03:01:04.027578  1973 raft_consensus.cc:556] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.032959  1324 tablet_bootstrap.cc:654] T 8935b23c661c4c1ca7bde7c893927502 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:04.061115  1918 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 154) took 1528 ms. Trace:
I20261120 03:01:04.061564  1918 rpcz_store.cc:276] 1120 03:01:02.533163 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:02.577679 (+ 44516us) service_pool.cc:224] Handling call
1120 03:01:03.901399 (+1323720us) ts_tablet_manager.cc:1197] Deleting tablet b549a06d15db449b851fe9549a963416
1120 03:01:03.901472 (+    73us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:03.965771 (+ 64299us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:04.004091 (+ 38320us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:04.061048 (+ 56957us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":2974,"tablet-delete.queue_time_us":1310037}
I20261120 03:01:04.063115   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet b549a06d15db449b851fe9549a963416 (table default.Table_61cb1e03b7e442a8822e32a2295866f3 [id=b5b565c2c2224675baada2ae9b42be54]) successfully deleted
I20261120 03:01:04.064321  1918 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 159) took 1467 ms. Trace:
I20261120 03:01:04.064700  1918 rpcz_store.cc:276] 1120 03:01:02.597315 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:02.599413 (+  2098us) service_pool.cc:224] Handling call
1120 03:01:04.063915 (+1464502us) ts_tablet_manager.cc:1197] Deleting tablet ad4e9e0de4954904842bfd5f15f530f0
1120 03:01:04.063967 (+    52us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:04.064269 (+   302us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1463884}
W20261120 03:01:04.065964   487 catalog_manager.cc:5061] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet ad4e9e0de4954904842bfd5f15f530f0 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet ad4e9e0de4954904842bfd5f15f530f0 already in progress: creating tablet
I20261120 03:01:04.066926  1918 tablet_replica.cc:333] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:01:04.067845  1918 raft_consensus.cc:2440] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:04.068440  1918 raft_consensus.cc:2469] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:04.072584  1918 ts_tablet_manager.cc:1962] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:04.073838  1324 tablet_bootstrap.cc:492] T 8935b23c661c4c1ca7bde7c893927502 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:01:04.074250  1324 ts_tablet_manager.cc:1449] T 8935b23c661c4c1ca7bde7c893927502 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.049s	user 0.028s	sys 0.016s
I20261120 03:01:04.076525  1324 raft_consensus.cc:366] T 8935b23c661c4c1ca7bde7c893927502 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.076675  1180 raft_consensus.cc:366] T 73c993944b0a4977b55d40ad63bb69e3 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.077122  1324 raft_consensus.cc:392] T 8935b23c661c4c1ca7bde7c893927502 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:04.077319  1180 raft_consensus.cc:392] T 73c993944b0a4977b55d40ad63bb69e3 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:04.077375  1324 raft_consensus.cc:787] T 8935b23c661c4c1ca7bde7c893927502 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:01:04.077584  1180 raft_consensus.cc:787] T 73c993944b0a4977b55d40ad63bb69e3 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:04.077966  1324 consensus_queue.cc:261] T 8935b23c661c4c1ca7bde7c893927502 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.079957  1324 ts_tablet_manager.cc:1480] T 8935b23c661c4c1ca7bde7c893927502 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.005s	user 0.005s	sys 0.000s
I20261120 03:01:04.081300  1324 tablet_bootstrap.cc:492] T 0c36259ff308455d986243c6b07d81a6 P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:01:04.085418   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00f7c7d95bae4d55be6b437701989ba3" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:04.086212   920 raft_consensus.cc:2628] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 823abdbd90ba47bbacad0f8dcff78234 in current term 1: Already voted for candidate 3d30c2428730481db9b0ffd279dd3440 in this term.
I20261120 03:01:04.090593   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ad49c8e00aca4a88a091baceae320477" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:04.091323   918 raft_consensus.cc:2062] T ad49c8e00aca4a88a091baceae320477 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 0
W20261120 03:01:04.092767  1012 leader_election.cc:343] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:04.093850  1324 tablet_bootstrap.cc:654] T 0c36259ff308455d986243c6b07d81a6 P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:04.110088   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ba85dab0d238487181ce5c1592a2c7fd" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:04.110790   777 raft_consensus.cc:3296] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:04.111775   777 raft_consensus.cc:2062] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
I20261120 03:01:04.112000  1324 tablet_bootstrap.cc:492] T 0c36259ff308455d986243c6b07d81a6 P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:01:04.112411  1324 ts_tablet_manager.cc:1449] T 0c36259ff308455d986243c6b07d81a6 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.031s	user 0.010s	sys 0.005s
I20261120 03:01:04.114709  1324 raft_consensus.cc:366] T 0c36259ff308455d986243c6b07d81a6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.115307  1324 raft_consensus.cc:392] T 0c36259ff308455d986243c6b07d81a6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:04.115527  1324 raft_consensus.cc:787] T 0c36259ff308455d986243c6b07d81a6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:01:04.116287  1324 consensus_queue.cc:261] T 0c36259ff308455d986243c6b07d81a6 P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.118415  1324 ts_tablet_manager.cc:1480] T 0c36259ff308455d986243c6b07d81a6 P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.006s	user 0.006s	sys 0.001s
I20261120 03:01:04.119264  1324 tablet_bootstrap.cc:492] T 8f1fda857e6c400e80d3d177a3d748ff P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:01:04.120380  1011 leader_election.cc:304] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:01:04.124653  1324 tablet_bootstrap.cc:654] T 8f1fda857e6c400e80d3d177a3d748ff P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:04.135145  1324 tablet_bootstrap.cc:492] T 8f1fda857e6c400e80d3d177a3d748ff P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:01:04.135588  1324 ts_tablet_manager.cc:1449] T 8f1fda857e6c400e80d3d177a3d748ff P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.016s	user 0.006s	sys 0.010s
I20261120 03:01:04.137862  1324 raft_consensus.cc:366] T 8f1fda857e6c400e80d3d177a3d748ff P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:04.138491  1324 raft_consensus.cc:392] T 8f1fda857e6c400e80d3d177a3d748ff P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:04.138727  1324 raft_consensus.cc:787] T 8f1fda857e6c400e80d3d177a3d748ff P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:01:04.139564  1918 ts_tablet_manager.cc:1975] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:04.151201  1918 log.cc:1199] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/c6a65c1a4a574961bec657ad7c9048e0
I20261120 03:01:04.152520  1918 ts_tablet_manager.cc:1996] T c6a65c1a4a574961bec657ad7c9048e0 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:01:04.157205  1918 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 156) took 1560 ms. Trace:
I20261120 03:01:04.164630  1974 raft_consensus.cc:3024] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:04.165086  1974 raft_consensus.cc:744] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Becoming Leader. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Running, Role: LEADER
I20261120 03:01:04.165836  1974 consensus_queue.cc:238] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:03.977934   493 catalog_manager.cc:5856] T 0179a59749a14168a840e9d4e610be1f P 3d30c2428730481db9b0ffd279dd3440 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66). New cstate: current_term: 1 leader_uuid: "3d30c2428730481db9b0ffd279dd3440" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: UNKNOWN } } }
I20261120 03:01:04.167304   493 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 0179a59749a14168a840e9d4e610be1f (table default.Table_cc56bf0ad8344f1a94d5da6e4d77e8c0 [id=b9034a083c7e4133a746b336335d3e42]). Expected version 1 got 0
I20261120 03:01:04.174357  1972 leader_election.cc:290] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Requested vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:04.174573  1975 leader_election.cc:290] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Requested vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:04.174901  1972 raft_consensus.cc:534] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:04.175060  1918 rpcz_store.cc:276] 1120 03:01:02.596503 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:02.606376 (+  9873us) service_pool.cc:224] Handling call
1120 03:01:04.066761 (+1460385us) ts_tablet_manager.cc:1197] Deleting tablet c6a65c1a4a574961bec657ad7c9048e0
1120 03:01:04.066811 (+    50us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:04.101100 (+ 34289us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:04.139420 (+ 38320us) tablet_metadata.cc:680] Metadata flushed
1120 03:01:04.157129 (+ 17709us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1459840}
I20261120 03:01:04.175400  1972 raft_consensus.cc:556] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.176730  1918 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:44896 (request call id 161) took 1484 ms. Trace:
I20261120 03:01:04.177066  1918 rpcz_store.cc:276] 1120 03:01:02.692533 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:02.692644 (+   111us) service_pool.cc:224] Handling call
1120 03:01:04.176256 (+1483612us) ts_tablet_manager.cc:1197] Deleting tablet 29e6a50c9e1f4b778737abdef9ad073a
1120 03:01:04.176336 (+    80us) ts_tablet_manager.cc:1117] Acquired tablet manager lock
1120 03:01:04.176659 (+   323us) inbound_call.cc:177] Queueing success response
Metrics: {"tablet-delete.queue_time_us":1467593}
I20261120 03:01:04.160981   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ba85dab0d238487181ce5c1592a2c7fd" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:01:04.178510   919 raft_consensus.cc:2628] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 823abdbd90ba47bbacad0f8dcff78234 in current term 1: Already voted for candidate 3d30c2428730481db9b0ffd279dd3440 in this term.
I20261120 03:01:04.178632   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet c6a65c1a4a574961bec657ad7c9048e0 (table default.Table_dd2f7617ff0e401b9b862d5991110dd5 [id=3955f41b6d4d41bf8ba2c81e4ac7f4a8]) successfully deleted
W20261120 03:01:04.179263   487 catalog_manager.cc:5061] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 29e6a50c9e1f4b778737abdef9ad073a because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 29e6a50c9e1f4b778737abdef9ad073a already in progress: creating tablet
I20261120 03:01:04.185002  1977 leader_election.cc:290] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Requested vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:04.185217  1973 leader_election.cc:290] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Requested vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:04.190233   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 0db7db25d779459ebde532f938493975 (table default.Table_9fe8b8c914534e4ca5602de7c1b1d280 [id=f3b5f7a4e5494e82a472d0025df96e0c]) successfully deleted
W20261120 03:01:04.191128   486 catalog_manager.cc:5061] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 5c3cfc12ab6041ce80b852177f2b6854 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 5c3cfc12ab6041ce80b852177f2b6854 already in progress: creating tablet
I20261120 03:01:04.191450  1915 tablet_replica.cc:333] T 1faba22e8e38418383b86e42e904a761 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:01:04.191933  1918 tablet_replica.cc:333] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:01:04.192536  1915 raft_consensus.cc:2440] T 1faba22e8e38418383b86e42e904a761 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:04.192871  1918 raft_consensus.cc:2440] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:01:04.193851  1918 raft_consensus.cc:2469] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:04.196744  1918 ts_tablet_manager.cc:1962] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:04.210711  1180 consensus_queue.cc:261] T 73c993944b0a4977b55d40ad63bb69e3 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.218391  1324 consensus_queue.cc:261] T 8f1fda857e6c400e80d3d177a3d748ff P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:04.220238  1324 ts_tablet_manager.cc:1480] T 8f1fda857e6c400e80d3d177a3d748ff P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.084s	user 0.024s	sys 0.023s
I20261120 03:01:04.221122  1324 tablet_bootstrap.cc:492] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:01:04.199704  1181 tablet_bootstrap.cc:492] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:04.242494  1181 ts_tablet_manager.cc:1449] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.299s	user 0.028s	sys 0.014s
I20261120 03:01:04.244961  1181 raft_consensus.cc:366] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:04.245680  1181 raft_consensus.cc:392] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:04.246111  1181 raft_consensus.cc:787] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:04.246778  1181 consensus_queue.cc:261] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:04.248957  1181 ts_tablet_manager.cc:1480] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.000s
I20261120 03:01:04.249848  1181 tablet_bootstrap.cc:492] T 5da3d2724dd34aed88e6b08983188004 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
W20261120 03:01:04.208724  1012 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 48379.4us
I20261120 03:01:04.256093   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3bc0f2f9cb7945ab9b0ead61c934f9dd" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:04.256877   918 raft_consensus.cc:2628] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 823abdbd90ba47bbacad0f8dcff78234 in current term 1: Already voted for candidate 3d30c2428730481db9b0ffd279dd3440 in this term.
I20261120 03:01:04.259912   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00f7c7d95bae4d55be6b437701989ba3" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:01:04.260617   918 raft_consensus.cc:2628] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 823abdbd90ba47bbacad0f8dcff78234 in current term 1: Already voted for candidate 3d30c2428730481db9b0ffd279dd3440 in this term.
I20261120 03:01:04.263069   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ad49c8e00aca4a88a091baceae320477" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:01:04.264865   918 raft_consensus.cc:3296] T ad49c8e00aca4a88a091baceae320477 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:04.266009   919 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "73c993944b0a4977b55d40ad63bb69e3" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440"
I20261120 03:01:04.266625   919 raft_consensus.cc:3296] T 73c993944b0a4977b55d40ad63bb69e3 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
W20261120 03:01:04.214198  1011 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 93897.3us
I20261120 03:01:04.268128  1918 ts_tablet_manager.cc:1975] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.3
I20261120 03:01:04.268481  1918 log.cc:1199] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/0db7db25d779459ebde532f938493975
I20261120 03:01:04.268668   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "73c993944b0a4977b55d40ad63bb69e3" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:04.269188  1918 ts_tablet_manager.cc:1996] T 0db7db25d779459ebde532f938493975 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:01:04.269178   777 raft_consensus.cc:3296] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:04.270049   777 raft_consensus.cc:2062] T 73c993944b0a4977b55d40ad63bb69e3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
I20261120 03:01:04.273440  1971 raft_consensus.cc:534] T 59cc37f7ef9c409493bd89d0d2097339 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:04.273887  1971 raft_consensus.cc:556] T 59cc37f7ef9c409493bd89d0d2097339 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.275926  1971 leader_election.cc:290] T 59cc37f7ef9c409493bd89d0d2097339 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:04.277938   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 73c993944b0a4977b55d40ad63bb69e3 (table default.Table_a96b0ac352c545da9b955502704e266b [id=1cdd4bbccdbf4449ba67b9aa7089a6fd]). Expected version 1 got 0
I20261120 03:01:04.279381   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "3bc0f2f9cb7945ab9b0ead61c934f9dd" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:04.280372   777 raft_consensus.cc:2628] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 823abdbd90ba47bbacad0f8dcff78234 in current term 1: Already voted for candidate 5beeefe8bef54caeac463d05d8c30fa6 in this term.
I20261120 03:01:04.280378   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 0179a59749a14168a840e9d4e610be1f (table default.Table_cc56bf0ad8344f1a94d5da6e4d77e8c0 [id=b9034a083c7e4133a746b336335d3e42]). Expected version 1 got 0
I20261120 03:01:04.283900  1971 raft_consensus.cc:534] T 0179a59749a14168a840e9d4e610be1f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:04.284341  1971 raft_consensus.cc:556] T 0179a59749a14168a840e9d4e610be1f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.285606  1324 tablet_bootstrap.cc:654] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:04.286039  1971 leader_election.cc:290] T 0179a59749a14168a840e9d4e610be1f P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:04.177541  1972 leader_election.cc:290] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:04.289058   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00f7c7d95bae4d55be6b437701989ba3" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:04.289636   777 raft_consensus.cc:3296] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:04.297236   777 raft_consensus.cc:2062] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
I20261120 03:01:04.300740   920 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "59cc37f7ef9c409493bd89d0d2097339" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:04.308127  1808 raft_consensus.cc:534] T 0c36259ff308455d986243c6b07d81a6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:04.308655  1808 raft_consensus.cc:556] T 0c36259ff308455d986243c6b07d81a6 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.311606  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0c36259ff308455d986243c6b07d81a6" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:04.312671   494 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet b6f0ff2246cd4772af47812b699e93e8 (table default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668]). Expected version 2 got 0
W20261120 03:01:04.313040   709 leader_election.cc:343] T 0c36259ff308455d986243c6b07d81a6 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:04.314006  1808 leader_election.cc:290] T 0c36259ff308455d986243c6b07d81a6 P 5beeefe8bef54caeac463d05d8c30fa6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:04.315588   494 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 73c993944b0a4977b55d40ad63bb69e3 (table default.Table_a96b0ac352c545da9b955502704e266b [id=1cdd4bbccdbf4449ba67b9aa7089a6fd]). Expected version 1 got 0
I20261120 03:01:04.316207  1324 tablet_bootstrap.cc:492] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:01:04.316565  1324 ts_tablet_manager.cc:1449] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.096s	user 0.009s	sys 0.000s
I20261120 03:01:04.318318   494 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 0179a59749a14168a840e9d4e610be1f (table default.Table_cc56bf0ad8344f1a94d5da6e4d77e8c0 [id=b9034a083c7e4133a746b336335d3e42]). Expected version 1 got 0
I20261120 03:01:04.318820   494 catalog_manager.cc:5956] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039) does not have the latest schema for tablet 5da3d2724dd34aed88e6b08983188004 (table default.Table_d2bd3060da1545548f422436c9c7c22e [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]). Expected version 2 got 0
I20261120 03:01:04.318766  1324 raft_consensus.cc:366] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:04.319453  1324 raft_consensus.cc:392] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:04.319693  1324 raft_consensus.cc:787] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5beeefe8bef54caeac463d05d8c30fa6, State: Initialized, Role: FOLLOWER
I20261120 03:01:04.320295  1324 consensus_queue.cc:261] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 5beeefe8bef54caeac463d05d8c30fa6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:04.321437   498 rpcz_store.cc:275] Call kudu.master.MasterService.AlterTable from 127.0.0.1:45640 (request call id 191) took 1677 ms. Trace:
I20261120 03:01:04.321621  1972 consensus_queue.cc:1049] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:04.322363  1324 ts_tablet_manager.cc:1480] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 5beeefe8bef54caeac463d05d8c30fa6: Time spent starting tablet: real 0.006s	user 0.006s	sys 0.001s
I20261120 03:01:04.323189  1324 tablet_bootstrap.cc:492] T 5ea1646a2ab84345a77fd7ffd249003a P 5beeefe8bef54caeac463d05d8c30fa6: Bootstrap starting.
I20261120 03:01:04.321753   498 rpcz_store.cc:276] 1120 03:01:02.643950 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:02.955189 (+311239us) service_pool.cc:224] Handling call
1120 03:01:02.956177 (+   988us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:01:02.957712 (+  1535us) catalog_manager.cc:3769] Apply alter schema
1120 03:01:03.115086 (+157374us) catalog_manager.cc:3801] Acquired catalog manager lock
1120 03:01:03.115393 (+   307us) catalog_manager.cc:3972] Updating metadata on disk
1120 03:01:03.505953 (+390560us) catalog_manager.cc:4003] Committing alterations to in-memory state
1120 03:01:04.321376 (+815423us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:03.116644 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:03.117049 (+   405us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:03.117072 (+    23us) write_op.cc:440] Acquired schema lock
1120 03:01:03.117086 (+    14us) tablet.cc:710] Decoding operations
1120 03:01:03.117350 (+   264us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:03.117385 (+    35us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:03.117404 (+    19us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:01:03.117553 (+   149us) tablet.cc:758] Row locks acquired
1120 03:01:03.117565 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:01:03.117670 (+   105us) write_op.cc:270] Start()
1120 03:01:03.117749 (+    79us) write_op.cc:276] Timestamp: P: 1795143663117601 usec, L: 0
1120 03:01:03.117764 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:01:03.118437 (+   673us) log.cc:844] Serialized 405 byte log entry
1120 03:01:03.491806 (+373369us) op_driver.cc:466] REPLICATION: finished
1120 03:01:03.494542 (+  2736us) write_op.cc:301] APPLY: starting
1120 03:01:03.494591 (+    49us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:03.494692 (+   101us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:03.494771 (+    79us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:03.494913 (+   142us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:03.495028 (+   115us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:01:03.495071 (+    43us) write_op.cc:312] APPLY: finished
1120 03:01:03.495252 (+   181us) log.cc:844] Serialized 23 byte log entry
1120 03:01:03.495392 (+   140us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.495445 (+    53us) write_op.cc:459] Released schema lock
1120 03:01:03.495474 (+    29us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:03.506291 (+ 10817us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.506318 (+    27us) write_op.cc:459] Released schema lock
Metrics: {"child_traces":[["op",{"apply.queue_time_us":2607,"apply.run_cpu_time_us":1150,"apply.run_wall_time_us":11707,"num_ops":1,"prepare.queue_time_us":93,"prepare.run_cpu_time_us":1776,"prepare.run_wall_time_us":1979,"replication_time_us":373961}]]}
I20261120 03:01:04.330281   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ad49c8e00aca4a88a091baceae320477" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:04.330806   777 raft_consensus.cc:3296] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:04.331624   777 raft_consensus.cc:2062] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
I20261120 03:01:04.331972   385 master-stress-test.cc:581] Killed test thread 5/10
I20261120 03:01:04.332218   385 master-stress-test.cc:579] Killing test thread 6/10
I20261120 03:01:04.332425   385 master-stress-test.cc:581] Killed test thread 6/10
I20261120 03:01:04.332607   385 master-stress-test.cc:579] Killing test thread 7/10
I20261120 03:01:04.333084   385 master-stress-test.cc:581] Killed test thread 7/10
I20261120 03:01:04.333273   385 master-stress-test.cc:579] Killing test thread 8/10
I20261120 03:01:04.342406  1324 tablet_bootstrap.cc:654] T 5ea1646a2ab84345a77fd7ffd249003a P 5beeefe8bef54caeac463d05d8c30fa6: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:04.345757  1915 raft_consensus.cc:2469] T 1faba22e8e38418383b86e42e904a761 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:04.361123   918 raft_consensus.cc:2062] T ad49c8e00aca4a88a091baceae320477 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
I20261120 03:01:04.361308  1180 ts_tablet_manager.cc:1480] T 73c993944b0a4977b55d40ad63bb69e3 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.468s	user 0.007s	sys 0.000s
I20261120 03:01:04.362346  1180 tablet_bootstrap.cc:492] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:01:04.365464   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 0db7db25d779459ebde532f938493975 (table default.Table_9fe8b8c914534e4ca5602de7c1b1d280 [id=f3b5f7a4e5494e82a472d0025df96e0c]) successfully deleted
W20261120 03:01:04.366164   487 catalog_manager.cc:5061] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 5c3cfc12ab6041ce80b852177f2b6854 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 5c3cfc12ab6041ce80b852177f2b6854 already in progress: creating tablet
I20261120 03:01:04.366816  1918 tablet_replica.cc:333] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:01:04.367774  1918 raft_consensus.cc:2440] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:01:04.368536  1180 tablet_bootstrap.cc:654] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:04.368474  1918 raft_consensus.cc:2469] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:04.369702   919 raft_consensus.cc:2062] T 73c993944b0a4977b55d40ad63bb69e3 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 823abdbd90ba47bbacad0f8dcff78234 in term 1
I20261120 03:01:04.371443  1918 ts_tablet_manager.cc:1962] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:04.376653  1074 raft_consensus.cc:3296] T 0179a59749a14168a840e9d4e610be1f P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:04.385747   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ad49c8e00aca4a88a091baceae320477" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:04.386613   776 raft_consensus.cc:2628] T ad49c8e00aca4a88a091baceae320477 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 823abdbd90ba47bbacad0f8dcff78234 in this term.
I20261120 03:01:04.393652  1915 ts_tablet_manager.cc:1962] T 1faba22e8e38418383b86e42e904a761 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:04.392033   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0179a59749a14168a840e9d4e610be1f" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:01:04.389097   857 consensus_peers.cc:183] SignalRequest failed: Illegal state: peer closed
W20261120 03:01:04.407887   857 leader_election.cc:343] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:01:04.413692   857 leader_election.cc:343] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:04.421370   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00f7c7d95bae4d55be6b437701989ba3" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:04.422127   776 raft_consensus.cc:2628] T 00f7c7d95bae4d55be6b437701989ba3 P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 823abdbd90ba47bbacad0f8dcff78234 in this term.
I20261120 03:01:04.426555   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ba85dab0d238487181ce5c1592a2c7fd" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6"
I20261120 03:01:04.427309   776 raft_consensus.cc:2628] T ba85dab0d238487181ce5c1592a2c7fd P 5beeefe8bef54caeac463d05d8c30fa6 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 823abdbd90ba47bbacad0f8dcff78234 in this term.
I20261120 03:01:04.436887   776 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "59cc37f7ef9c409493bd89d0d2097339" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:04.459838  1992 tablet.cc:1872] T b6f0ff2246cd4772af47812b699e93e8 P 5beeefe8bef54caeac463d05d8c30fa6: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 2
I20261120 03:01:04.428758  1972 raft_consensus.cc:534] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:04.429533  1012 leader_election.cc:304] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:01:04.433285  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ad49c8e00aca4a88a091baceae320477" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:04.434317  1073 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00f7c7d95bae4d55be6b437701989ba3" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:01:04.472343  1999 tablet.cc:1872] T c6fea104598e4f6d876f96675f2da816 P 5beeefe8bef54caeac463d05d8c30fa6: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20261120 03:01:04.488145   777 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0179a59749a14168a840e9d4e610be1f" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5beeefe8bef54caeac463d05d8c30fa6" is_pre_election: true
I20261120 03:01:04.443166  1918 ts_tablet_manager.cc:1975] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:04.490424  1918 log.cc:1199] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/1faba22e8e38418383b86e42e904a761
I20261120 03:01:04.491029   495 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 0179a59749a14168a840e9d4e610be1f (table default.Table_cc56bf0ad8344f1a94d5da6e4d77e8c0 [id=b9034a083c7e4133a746b336335d3e42]). Expected version 1 got 0
I20261120 03:01:04.491295  1918 ts_tablet_manager.cc:1996] T 1faba22e8e38418383b86e42e904a761 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:01:04.435292  1076 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "ba85dab0d238487181ce5c1592a2c7fd" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:01:04.494062   497 catalog_manager.cc:6409] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: default.Table_3e877edca926443fba86366475176a93 [id=93a0027d82064a208e2535349e9b042d] alter complete (version 2)
I20261120 03:01:04.495254   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 1faba22e8e38418383b86e42e904a761 (table default.Table_47101d55d8f6485c816a02a54fda67a8 [id=252d229ff1334cd99ec99e5c921031bb]) successfully deleted
I20261120 03:01:04.476840   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "0c36259ff308455d986243c6b07d81a6" candidate_uuid: "5beeefe8bef54caeac463d05d8c30fa6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
W20261120 03:01:04.482719  1012 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 53230.3us
W20261120 03:01:04.498178   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 84513.5us
I20261120 03:01:04.500671  1073 raft_consensus.cc:2628] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 823abdbd90ba47bbacad0f8dcff78234 in this term.
I20261120 03:01:04.484603  1931 raft_consensus.cc:534] T bed7910ab7c140199de04e306c532969 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:04.501381  1972 raft_consensus.cc:556] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.503367  1972 leader_election.cc:290] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:04.503952  1075 raft_consensus.cc:2628] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 3d30c2428730481db9b0ffd279dd3440 in current term 1: Already voted for candidate 823abdbd90ba47bbacad0f8dcff78234 in this term.
I20261120 03:01:04.504621  1971 raft_consensus.cc:3024] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:04.505254  1971 raft_consensus.cc:744] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Becoming Leader. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Running, Role: LEADER
I20261120 03:01:04.506063  1971 consensus_queue.cc:238] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.503145  1931 raft_consensus.cc:556] T bed7910ab7c140199de04e306c532969 P 5beeefe8bef54caeac463d05d8c30fa6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.508605   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9ea5f0b680bd401bbc2b69d780e1ef50" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:04.517833  2001 tablet.cc:1872] T c6fea104598e4f6d876f96675f2da816 P 823abdbd90ba47bbacad0f8dcff78234: Alter schema from (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 0 to (
    10:key INT32 NOT NULL,
    PRIMARY KEY (key)
) version 1
I20261120 03:01:04.524204   902 tablet_service.cc:1495] Processing CreateTablet for tablet cfb764b70e7a4e8f8a0d5269005d6ab9 (DEFAULT_TABLE table=default.Table_736330745d7048c6b1fad681e15ba40d [id=66675f42cb574eb6b4f3c14cf00176e6]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:04.525297   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet cfb764b70e7a4e8f8a0d5269005d6ab9. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:04.527779  1012 leader_election.cc:304] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:01:04.531241  1893 raft_consensus.cc:3024] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:04.531710  1893 raft_consensus.cc:744] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Becoming Leader. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Running, Role: LEADER
I20261120 03:01:04.532475  1893 consensus_queue.cc:238] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.534389   502 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: Sending DeleteTablet for 3 replicas of tablet b3f7103b59d443f9bc739083b8e023ce
I20261120 03:01:04.535874   502 rpcz_store.cc:275] Call kudu.master.MasterService.DeleteTable from 127.0.0.1:45640 (request call id 189) took 2585 ms. Trace:
I20261120 03:01:04.544544  1180 tablet_bootstrap.cc:492] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:01:04.544997  1180 ts_tablet_manager.cc:1449] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.183s	user 0.039s	sys 0.012s
W20261120 03:01:04.545397   497 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.67:53803 (request call id 22) took 2768 ms (client timeout 1000 ms). Trace:
I20261120 03:01:04.546190  1324 tablet_bootstrap.cc:492] T 5ea1646a2ab84345a77fd7ffd249003a P 5beeefe8bef54caeac463d05d8c30fa6: No bootstrap required, opened a new log
I20261120 03:01:04.546586  1324 ts_tablet_manager.cc:1449] T 5ea1646a2ab84345a77fd7ffd249003a P 5beeefe8bef54caeac463d05d8c30fa6: Time spent bootstrapping tablet: real 0.224s	user 0.026s	sys 0.029s
W20261120 03:01:04.545730   497 rpcz_store.cc:269] 1120 03:01:01.776862 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:01.784411 (+  7549us) service_pool.cc:224] Handling call
1120 03:01:02.318170 (+533759us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:02.318197 (+    27us) write_op.cc:459] Released schema lock
1120 03:01:04.545317 (+2227120us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:02.221296 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:02.221911 (+   615us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:02.221933 (+    22us) write_op.cc:440] Acquired schema lock
1120 03:01:02.221945 (+    12us) tablet.cc:710] Decoding operations
1120 03:01:02.222221 (+   276us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:02.222259 (+    38us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:02.222280 (+    21us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:01:02.222418 (+   138us) tablet.cc:758] Row locks acquired
1120 03:01:02.222430 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:01:02.222535 (+   105us) write_op.cc:270] Start()
1120 03:01:02.222620 (+    85us) write_op.cc:276] Timestamp: P: 1795143662222448 usec, L: 0
1120 03:01:02.222634 (+    14us) op_driver.cc:350] REPLICATION: starting
1120 03:01:02.223053 (+   419us) log.cc:844] Serialized 356 byte log entry
1120 03:01:02.232450 (+  9397us) op_driver.cc:466] REPLICATION: finished
1120 03:01:02.232666 (+   216us) write_op.cc:301] APPLY: starting
1120 03:01:02.232719 (+    53us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:02.232807 (+    88us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:02.232899 (+    92us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:02.233038 (+   139us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:02.233154 (+   116us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:01:02.233195 (+    41us) write_op.cc:312] APPLY: finished
1120 03:01:02.233351 (+   156us) log.cc:844] Serialized 23 byte log entry
1120 03:01:02.233474 (+   123us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:02.233532 (+    58us) write_op.cc:459] Released schema lock
1120 03:01:02.233562 (+    30us) write_op.cc:346] FINISH: Updating metrics
Related trace 'op':
1120 03:01:02.545246 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:02.545659 (+   413us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:02.545681 (+    22us) write_op.cc:440] Acquired schema lock
1120 03:01:02.545694 (+    13us) tablet.cc:710] Decoding operations
1120 03:01:02.545951 (+   257us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:02.545985 (+    34us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:02.546006 (+    21us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:01:02.546176 (+   170us) tablet.cc:758] Row locks acquired
1120 03:01:02.546188 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:01:02.546292 (+   104us) write_op.cc:270] Start()
1120 03:01:02.546368 (+    76us) write_op.cc:276] Timestamp: P: 1795143662546230 usec, L: 0
1120 03:01:02.546382 (+    14us) op_driver.cc:350] REPLICATION: starting
1120 03:01:02.546802 (+   420us) log.cc:844] Serialized 356 byte log entry
1120 03:01:03.320924 (+774122us) op_driver.cc:466] REPLICATION: finished
1120 03:01:03.322109 (+  1185us) write_op.cc:301] APPLY: starting
1120 03:01:03.322167 (+    58us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:03.322277 (+   110us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:03.322367 (+    90us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:03.322525 (+   158us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:03.322656 (+   131us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:01:03.322696 (+    40us) write_op.cc:312] APPLY: finished
1120 03:01:03.322879 (+   183us) log.cc:844] Serialized 23 byte log entry
1120 03:01:03.323043 (+   164us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.323098 (+    55us) write_op.cc:459] Released schema lock
1120 03:01:03.323129 (+    31us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"reported_tablets":19,"child_traces":[["op",{"apply.queue_time_us":82,"apply.run_cpu_time_us":1059,"apply.run_wall_time_us":1058,"num_ops":1,"prepare.queue_time_us":90,"prepare.run_cpu_time_us":1893,"prepare.run_wall_time_us":3648,"raft.queue_time_us":941,"raft.run_cpu_time_us":1329,"raft.run_wall_time_us":1329,"replication_time_us":9723}],["op",{"apply.queue_time_us":999,"apply.run_cpu_time_us":1314,"apply.run_wall_time_us":2342,"mutex_wait_us":12641,"num_ops":1,"prepare.queue_time_us":14826,"prepare.run_cpu_time_us":1767,"prepare.run_wall_time_us":3025,"replication_time_us":774440,"thread_start_us":25039,"threads_started":1}]]}
I20261120 03:01:04.551442  1046 tablet_service.cc:1542] Processing DeleteTablet for tablet b3f7103b59d443f9bc739083b8e023ce with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:01:02 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:04.552182   753 tablet_service.cc:1542] Processing DeleteTablet for tablet b3f7103b59d443f9bc739083b8e023ce with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:01:02 UTC) from {username='slave'} at 127.0.0.1:52644
I20261120 03:01:04.500797  1181 tablet_bootstrap.cc:654] T 5da3d2724dd34aed88e6b08983188004 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:04.536162   502 rpcz_store.cc:276] 1120 03:01:01.950397 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:02.535971 (+585574us) service_pool.cc:224] Handling call
1120 03:01:02.562332 (+ 26361us) catalog_manager.cc:2437] Looking up, locking, and authorizing table
1120 03:01:02.563916 (+  1584us) catalog_manager.cc:2859] Modifying in-memory table state
1120 03:01:02.564038 (+   122us) catalog_manager.cc:2868] Locking tablets
1120 03:01:02.565115 (+  1077us) catalog_manager.cc:2882] Removing table and tablets from system table
1120 03:01:03.366015 (+800900us) catalog_manager.cc:2900] Removing table from by-name map
1120 03:01:03.366268 (+   253us) catalog_manager.cc:2916] Committing in-memory state
1120 03:01:04.534276 (+1168008us) catalog_manager.cc:2920] Aborting table tasks
1120 03:01:04.535814 (+  1538us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:02.618394 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:02.618787 (+   393us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:02.618811 (+    24us) write_op.cc:440] Acquired schema lock
1120 03:01:02.618824 (+    13us) tablet.cc:710] Decoding operations
1120 03:01:02.619170 (+   346us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:02.619208 (+    38us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:02.619230 (+    22us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:01:02.619498 (+   268us) tablet.cc:758] Row locks acquired
1120 03:01:02.619511 (+    13us) write_op.cc:260] PREPARE: finished
1120 03:01:02.619626 (+   115us) write_op.cc:270] Start()
1120 03:01:02.619854 (+   228us) write_op.cc:276] Timestamp: P: 1795143662619553 usec, L: 0
1120 03:01:02.619871 (+    17us) op_driver.cc:350] REPLICATION: starting
1120 03:01:02.620476 (+   605us) log.cc:844] Serialized 759 byte log entry
1120 03:01:03.355691 (+735215us) op_driver.cc:466] REPLICATION: finished
1120 03:01:03.363696 (+  8005us) write_op.cc:301] APPLY: starting
1120 03:01:03.363755 (+    59us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:03.363861 (+   106us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:03.364084 (+   223us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:03.364320 (+   236us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:03.364460 (+   140us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
1120 03:01:03.364510 (+    50us) write_op.cc:312] APPLY: finished
1120 03:01:03.364700 (+   190us) log.cc:844] Serialized 29 byte log entry
1120 03:01:03.364833 (+   133us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.364889 (+    56us) write_op.cc:459] Released schema lock
1120 03:01:03.364917 (+    28us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"child_traces":[["op",{"apply.queue_time_us":7876,"apply.run_cpu_time_us":1396,"apply.run_wall_time_us":1397,"num_ops":2,"prepare.queue_time_us":11350,"prepare.run_cpu_time_us":2113,"prepare.run_wall_time_us":2286,"replication_time_us":735741,"thread_start_us":9874,"threads_started":1}]]}
I20261120 03:01:04.554239   903 tablet_service.cc:1542] Processing DeleteTablet for tablet b3f7103b59d443f9bc739083b8e023ce with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:01:02 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:01:04.554953  1918 tablet_replica.cc:333] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:01:04.555696   385 master-stress-test.cc:581] Killed test thread 8/10
I20261120 03:01:04.555949   385 master-stress-test.cc:579] Killing test thread 9/10
I20261120 03:01:04.556254   385 master-stress-test.cc:581] Killed test thread 9/10
I20261120 03:01:04.556442   385 master-stress-test.cc:579] Killing test thread 10/10
I20261120 03:01:04.556697   385 master-stress-test.cc:581] Killed test thread 10/10
I20261120 03:01:04.557015   385 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu with pid 667
W20261120 03:01:04.564102   501 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.65:40295 (request call id 36) took 3276 ms (client timeout 1000 ms). Trace:
W20261120 03:01:04.564467   501 rpcz_store.cc:269] 1120 03:01:01.287467 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:01.295509 (+  8042us) service_pool.cc:224] Handling call
1120 03:01:04.564013 (+3268504us) inbound_call.cc:177] Queueing success response
Metrics: {"reported_tablets":36}
I20261120 03:01:04.568802   903 tablet_service.cc:1542] Processing DeleteTablet for tablet b549a06d15db449b851fe9549a963416 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:54 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:01:04.569886   903 tablet_service.cc:1542] Processing DeleteTablet for tablet c6a65c1a4a574961bec657ad7c9048e0 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:57 UTC) from {username='slave'} at 127.0.0.1:44896
I20261120 03:01:04.571012   903 tablet_service.cc:1542] Processing DeleteTablet for tablet 0db7db25d779459ebde532f938493975 with delete_type TABLET_DATA_DELETED (replaced by tablet 2936e79caca943258b39849505c59f50) from {username='slave'} at 127.0.0.1:44896
I20261120 03:01:04.572067   903 tablet_service.cc:1542] Processing DeleteTablet for tablet a1192a7f050a4f5387a30b2216dec5f0 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:01:00 UTC) from {username='slave'} at 127.0.0.1:44896
W20261120 03:01:04.574030   493 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.66:36141 (request call id 32) took 1656 ms (client timeout 1000 ms). Trace:
W20261120 03:01:04.574410   493 rpcz_store.cc:269] 1120 03:01:02.917951 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:02.965811 (+ 47860us) service_pool.cc:224] Handling call
1120 03:01:04.573964 (+1608153us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:04.225409 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:04.225839 (+   430us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:04.225865 (+    26us) write_op.cc:440] Acquired schema lock
1120 03:01:04.225878 (+    13us) tablet.cc:710] Decoding operations
1120 03:01:04.226196 (+   318us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:04.226238 (+    42us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:04.226259 (+    21us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:01:04.226409 (+   150us) tablet.cc:758] Row locks acquired
1120 03:01:04.226421 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:01:04.226534 (+   113us) write_op.cc:270] Start()
1120 03:01:04.226614 (+    80us) write_op.cc:276] Timestamp: P: 1795143664226453 usec, L: 0
1120 03:01:04.226628 (+    14us) op_driver.cc:350] REPLICATION: starting
1120 03:01:04.227126 (+   498us) log.cc:844] Serialized 527 byte log entry
1120 03:01:04.235525 (+  8399us) op_driver.cc:466] REPLICATION: finished
1120 03:01:04.235941 (+   416us) write_op.cc:301] APPLY: starting
1120 03:01:04.236015 (+    74us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:04.236109 (+    94us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:04.236201 (+    92us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:04.236357 (+   156us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:04.236491 (+   134us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:01:04.236531 (+    40us) write_op.cc:312] APPLY: finished
1120 03:01:04.236712 (+   181us) log.cc:844] Serialized 23 byte log entry
1120 03:01:04.236857 (+   145us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:04.236914 (+    57us) write_op.cc:459] Released schema lock
1120 03:01:04.236942 (+    28us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:04.238521 (+  1579us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:04.238545 (+    24us) write_op.cc:459] Released schema lock
Metrics: {"mutex_wait_us":12195,"reported_tablets":23,"child_traces":[["op",{"apply.queue_time_us":261,"apply.run_cpu_time_us":1228,"apply.run_wall_time_us":1228,"mutex_wait_us":11003,"num_ops":1,"prepare.queue_time_us":55840,"prepare.run_cpu_time_us":2128,"prepare.run_wall_time_us":13112,"raft.queue_time_us":11493,"raft.run_cpu_time_us":1644,"raft.run_wall_time_us":1644,"replication_time_us":8785}]]}
W20261120 03:01:04.576794   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 55919.3us
I20261120 03:01:04.584748   496 catalog_manager.cc:6409] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: default.Table_2a81e9814fe34f9bb1902e71fd82fd01 [id=e540afdca1234f30bf1950448be502df] alter complete (version 1)
I20261120 03:01:04.599254  1011 leader_election.cc:304] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:01:04.600394  1011 leader_election.cc:304] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440
W20261120 03:01:04.600929   496 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.67:53803 (request call id 23) took 1265 ms (client timeout 1000 ms). Trace:
I20261120 03:01:04.604599  1051 tablet_service.cc:1542] Processing DeleteTablet for tablet 0db7db25d779459ebde532f938493975 with delete_type TABLET_DATA_DELETED (replaced by tablet 2936e79caca943258b39849505c59f50) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:04.604919  1059 tablet_service.cc:1542] Processing DeleteTablet for tablet 014bd9c83be34c31b90fe4bcf31f1649 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:54 UTC) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:04.605545  1046 tablet_service.cc:1542] Processing DeleteTablet for tablet b549a06d15db449b851fe9549a963416 with delete_type TABLET_DATA_DELETED (Table deleted at 2026-11-20 03:00:54 UTC) from {username='slave'} at 127.0.0.1:50744
W20261120 03:01:04.607398  1012 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.66:42987, user_credentials={real_user=slave}} blocked reactor thread for 79691.5us
W20261120 03:01:04.613453   496 rpcz_store.cc:269] 1120 03:01:03.335199 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:03.352427 (+ 17228us) service_pool.cc:224] Handling call
1120 03:01:03.369367 (+ 16940us) spinlock_profiling.cc:245] Waited 7.56 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07d9ce 00007f155abf0174 00007f155abf0102 00007f155abf0031 00007f155abeffac 00007f155abef262 00007f1558e990bc 00007f1558e984f6
1120 03:01:04.600853 (+1231486us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:03.579089 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:03.579472 (+   383us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:03.579493 (+    21us) write_op.cc:440] Acquired schema lock
1120 03:01:03.579504 (+    11us) tablet.cc:710] Decoding operations
1120 03:01:03.579786 (+   282us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:03.579824 (+    38us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:03.579845 (+    21us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:01:03.580024 (+   179us) tablet.cc:758] Row locks acquired
1120 03:01:03.580037 (+    13us) write_op.cc:260] PREPARE: finished
1120 03:01:03.580152 (+   115us) write_op.cc:270] Start()
1120 03:01:03.580231 (+    79us) write_op.cc:276] Timestamp: P: 1795143663580084 usec, L: 0
1120 03:01:03.580247 (+    16us) op_driver.cc:350] REPLICATION: starting
1120 03:01:03.580741 (+   494us) log.cc:844] Serialized 527 byte log entry
1120 03:01:03.686932 (+106191us) op_driver.cc:466] REPLICATION: finished
1120 03:01:03.689109 (+  2177us) write_op.cc:301] APPLY: starting
1120 03:01:03.689157 (+    48us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:03.689251 (+    94us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:03.689332 (+    81us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:03.689470 (+   138us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:03.689618 (+   148us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:01:03.689662 (+    44us) write_op.cc:312] APPLY: finished
1120 03:01:03.689817 (+   155us) log.cc:844] Serialized 23 byte log entry
1120 03:01:03.689960 (+   143us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.690013 (+    53us) write_op.cc:459] Released schema lock
1120 03:01:03.690046 (+    33us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:03.692559 (+  2513us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.692584 (+    25us) write_op.cc:459] Released schema lock
Related trace 'op':
1120 03:01:03.779615 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:03.780062 (+   447us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:03.780087 (+    25us) write_op.cc:440] Acquired schema lock
1120 03:01:03.780101 (+    14us) tablet.cc:710] Decoding operations
1120 03:01:03.780380 (+   279us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:03.780416 (+    36us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:03.780437 (+    21us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:01:03.780583 (+   146us) tablet.cc:758] Row locks acquired
1120 03:01:03.780595 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:01:03.780713 (+   118us) write_op.cc:270] Start()
1120 03:01:03.780797 (+    84us) write_op.cc:276] Timestamp: P: 1795143663780628 usec, L: 0
1120 03:01:03.780812 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:01:03.781307 (+   495us) log.cc:844] Serialized 356 byte log entry
1120 03:01:03.884926 (+103619us) op_driver.cc:466] REPLICATION: finished
1120 03:01:03.885553 (+   627us) write_op.cc:301] APPLY: starting
1120 03:01:03.885617 (+    64us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:03.885728 (+   111us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:03.885839 (+   111us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:03.886031 (+   192us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:03.886206 (+   175us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:01:03.886261 (+    55us) write_op.cc:312] APPLY: finished
1120 03:01:03.886469 (+   208us) log.cc:844] Serialized 23 byte log entry
1120 03:01:03.886635 (+   166us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.886703 (+    68us) write_op.cc:459] Released schema lock
1120 03:01:03.886743 (+    40us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:03.886994 (+   251us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.887020 (+    26us) write_op.cc:459] Released schema lock
Metrics: {"reported_tablets":23,"spinlock_wait_cycles":17391488,"child_traces":[["op",{"apply.queue_time_us":2045,"apply.run_cpu_time_us":1162,"apply.run_wall_time_us":3412,"num_ops":1,"prepare.queue_time_us":116,"prepare.run_cpu_time_us":1863,"prepare.run_wall_time_us":1862,"replication_time_us":106612}],["op",{"apply.queue_time_us":457,"apply.run_cpu_time_us":1405,"apply.run_wall_time_us":1397,"num_ops":1,"prepare.queue_time_us":155,"prepare.run_cpu_time_us":2010,"prepare.run_wall_time_us":2011,"raft.queue_time_us":1369,"raft.run_cpu_time_us":1644,"raft.run_wall_time_us":1829,"replication_time_us":103997,"spinlock_wait_cycles":674688}]]}
I20261120 03:01:04.633003  1181 tablet_bootstrap.cc:492] T 5da3d2724dd34aed88e6b08983188004 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:04.633581  1181 ts_tablet_manager.cc:1449] T 5da3d2724dd34aed88e6b08983188004 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.384s	user 0.011s	sys 0.000s
I20261120 03:01:04.634013  1983 raft_consensus.cc:3296] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:04.634896  1983 raft_consensus.cc:2966] T 3bc0f2f9cb7945ab9b0ead61c934f9dd P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:04.635353  1972 raft_consensus.cc:3024] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:04.635789  1972 raft_consensus.cc:744] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Becoming Leader. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Running, Role: LEADER
I20261120 03:01:04.636106  1181 raft_consensus.cc:366] T 5da3d2724dd34aed88e6b08983188004 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:04.636979  1972 consensus_queue.cc:238] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.638358  1975 consensus_queue.cc:1049] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:04.636719  1181 raft_consensus.cc:392] T 5da3d2724dd34aed88e6b08983188004 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:04.639048  1181 raft_consensus.cc:787] T 5da3d2724dd34aed88e6b08983188004 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:04.639632  1181 consensus_queue.cc:261] T 5da3d2724dd34aed88e6b08983188004 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:04.646574  1181 ts_tablet_manager.cc:1480] T 5da3d2724dd34aed88e6b08983188004 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.013s	user 0.005s	sys 0.000s
I20261120 03:01:04.647497  1181 tablet_bootstrap.cc:492] T 26f8ba69516c447dbd91e0d87cd519d3 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
W20261120 03:01:04.650825  1012 leader_election.cc:343] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:04.655483  1975 consensus_queue.cc:1049] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:04.657289  1975 consensus_queue.cc:1049] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:04.651502  1893 raft_consensus.cc:534] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:04.658237  1893 raft_consensus.cc:556] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:04.660156  1893 leader_election.cc:290] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987)
W20261120 03:01:04.654462  1011 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 52976.3us
I20261120 03:01:04.668221  1181 tablet_bootstrap.cc:654] T 26f8ba69516c447dbd91e0d87cd519d3 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:04.668541  1011 leader_election.cc:304] T 59cc37f7ef9c409493bd89d0d2097339 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:01:04.670311  1011 leader_election.cc:304] T 0179a59749a14168a840e9d4e610be1f P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:01:04.671124  1986 raft_consensus.cc:2966] T 59cc37f7ef9c409493bd89d0d2097339 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:04.671804  1986 raft_consensus.cc:2966] T 0179a59749a14168a840e9d4e610be1f P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:04.678597   500 catalog_manager.cc:6409] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: default.Table_fc294aebad394eb1b7c3ddd38d9f3a51 [id=606c1f1f5a8541e5be8cff24b2a4b668] alter complete (version 2)
I20261120 03:01:04.683312  1181 tablet_bootstrap.cc:492] T 26f8ba69516c447dbd91e0d87cd519d3 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:04.683929  1181 ts_tablet_manager.cc:1449] T 26f8ba69516c447dbd91e0d87cd519d3 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.037s	user 0.007s	sys 0.005s
I20261120 03:01:04.687268  1181 raft_consensus.cc:366] T 26f8ba69516c447dbd91e0d87cd519d3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.687860  1181 raft_consensus.cc:392] T 26f8ba69516c447dbd91e0d87cd519d3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:04.688117  1181 raft_consensus.cc:787] T 26f8ba69516c447dbd91e0d87cd519d3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:04.687973  1055 tablet_service.cc:1495] Processing CreateTablet for tablet cfb764b70e7a4e8f8a0d5269005d6ab9 (DEFAULT_TABLE table=default.Table_736330745d7048c6b1fad681e15ba40d [id=66675f42cb574eb6b4f3c14cf00176e6]), partition=RANGE (key) PARTITION UNBOUNDED
W20261120 03:01:04.682051   499 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.65:40295 (request call id 37) took 2188 ms (client timeout 1000 ms). Trace:
I20261120 03:01:04.689047  1055 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet cfb764b70e7a4e8f8a0d5269005d6ab9. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:04.688710  1181 consensus_queue.cc:261] T 26f8ba69516c447dbd91e0d87cd519d3 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.690567  1181 ts_tablet_manager.cc:1480] T 26f8ba69516c447dbd91e0d87cd519d3 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.006s	user 0.001s	sys 0.004s
I20261120 03:01:04.691370  1181 tablet_bootstrap.cc:492] T 4da5eb2f2f114cc792488c5c74bb1414 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:04.697180   903 tablet_service.cc:1542] Processing DeleteTablet for tablet 1faba22e8e38418383b86e42e904a761 with delete_type TABLET_DATA_DELETED (replaced by tablet 4ebd4502672049d9ae09d4446e1a3447) from {username='slave'} at 127.0.0.1:44896
W20261120 03:01:04.698745   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 120894us
W20261120 03:01:04.689407   499 rpcz_store.cc:269] 1120 03:01:02.493526 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:02.513299 (+ 19773us) service_pool.cc:224] Handling call
1120 03:01:02.568540 (+ 55241us) spinlock_profiling.cc:245] Waited 25.8 ms on lock 0x7b5400030554. stack: 00007f155401f023 00007f155401ef20 00007f1553bba574 00007f155d7d7964 00007f155d7d78f9 00007f155b148510 00007f155b1e914a 00007f155cf7fcfc 00007f155d07d9ce 00007f155abf0174 00007f155abf0102 00007f155abf0031 00007f155abeffac 00007f155abef262 00007f1558e990bc 00007f1558e984f6
1120 03:01:03.962565 (+1394025us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.962590 (+    25us) write_op.cc:459] Released schema lock
1120 03:01:04.681976 (+719386us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:03.937857 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:03.938289 (+   432us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:03.938314 (+    25us) write_op.cc:440] Acquired schema lock
1120 03:01:03.938328 (+    14us) tablet.cc:710] Decoding operations
1120 03:01:03.938639 (+   311us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:03.938679 (+    40us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:03.938701 (+    22us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:01:03.938965 (+   264us) tablet.cc:758] Row locks acquired
1120 03:01:03.938978 (+    13us) write_op.cc:260] PREPARE: finished
1120 03:01:03.939088 (+   110us) write_op.cc:270] Start()
1120 03:01:03.939181 (+    93us) write_op.cc:276] Timestamp: P: 1795143663939037 usec, L: 0
1120 03:01:03.939196 (+    15us) op_driver.cc:350] REPLICATION: starting
1120 03:01:03.939654 (+   458us) log.cc:844] Serialized 901 byte log entry
1120 03:01:03.949773 (+ 10119us) op_driver.cc:466] REPLICATION: finished
1120 03:01:03.950130 (+   357us) write_op.cc:301] APPLY: starting
1120 03:01:03.950200 (+    70us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:03.950315 (+   115us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:03.950419 (+   104us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:03.950657 (+   238us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:03.950787 (+   130us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=2
1120 03:01:03.950830 (+    43us) write_op.cc:312] APPLY: finished
1120 03:01:03.951007 (+   177us) log.cc:844] Serialized 29 byte log entry
1120 03:01:03.951153 (+   146us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:03.951230 (+    77us) write_op.cc:459] Released schema lock
1120 03:01:03.951263 (+    33us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"mutex_wait_us":129420,"reported_tablets":39,"spinlock_wait_cycles":59396480,"child_traces":[["op",{"apply.queue_time_us":193,"apply.run_cpu_time_us":1308,"apply.run_wall_time_us":1308,"num_ops":2,"prepare.queue_time_us":97,"prepare.run_cpu_time_us":2122,"prepare.run_wall_time_us":2123,"raft.queue_time_us":1293,"raft.run_cpu_time_us":1566,"raft.run_wall_time_us":1563,"replication_time_us":10476}]]}
W20261120 03:01:04.707602   500 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.67:53803 (request call id 24) took 840 ms (client timeout 1000 ms). Trace:
I20261120 03:01:04.707285   918 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9e42a416be884341994d2d5395286798" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:04.709905  1180 raft_consensus.cc:366] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
W20261120 03:01:04.708000   500 rpcz_store.cc:269] 1120 03:01:03.867188 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:03.868407 (+  1219us) service_pool.cc:224] Handling call
1120 03:01:04.678229 (+809822us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:04.678256 (+    27us) write_op.cc:459] Released schema lock
1120 03:01:04.707543 (+ 29287us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:04.640967 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:04.641394 (+   427us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:04.641418 (+    24us) write_op.cc:440] Acquired schema lock
1120 03:01:04.641431 (+    13us) tablet.cc:710] Decoding operations
1120 03:01:04.641698 (+   267us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:04.641738 (+    40us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:04.641761 (+    23us) tablet.cc:742] Acquiring locks for 1 operations
1120 03:01:04.641914 (+   153us) tablet.cc:758] Row locks acquired
1120 03:01:04.641926 (+    12us) write_op.cc:260] PREPARE: finished
1120 03:01:04.642041 (+   115us) write_op.cc:270] Start()
1120 03:01:04.642153 (+   112us) write_op.cc:276] Timestamp: P: 1795143664641969 usec, L: 0
1120 03:01:04.642169 (+    16us) op_driver.cc:350] REPLICATION: starting
1120 03:01:04.642619 (+   450us) log.cc:844] Serialized 356 byte log entry
1120 03:01:04.658362 (+ 15743us) op_driver.cc:466] REPLICATION: finished
1120 03:01:04.658678 (+   316us) write_op.cc:301] APPLY: starting
1120 03:01:04.658743 (+    65us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:04.658838 (+    95us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:04.658943 (+   105us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:04.659106 (+   163us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:04.659233 (+   127us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=1
1120 03:01:04.659279 (+    46us) write_op.cc:312] APPLY: finished
1120 03:01:04.659457 (+   178us) log.cc:844] Serialized 23 byte log entry
1120 03:01:04.659607 (+   150us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:04.659672 (+    65us) write_op.cc:459] Released schema lock
1120 03:01:04.659703 (+    31us) write_op.cc:346] FINISH: Updating metrics
Metrics: {"mutex_wait_us":13505,"reported_tablets":32,"child_traces":[["op",{"apply.queue_time_us":167,"apply.run_cpu_time_us":1193,"apply.run_wall_time_us":1193,"num_ops":1,"prepare.queue_time_us":12751,"prepare.run_cpu_time_us":1972,"prepare.run_wall_time_us":1973,"raft.queue_time_us":3472,"raft.run_cpu_time_us":1611,"raft.run_wall_time_us":1611,"replication_time_us":16096}]]}
I20261120 03:01:04.710587  1180 raft_consensus.cc:392] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:04.710825  1180 raft_consensus.cc:787] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:04.711441  1918 raft_consensus.cc:2440] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:04.711984  1918 raft_consensus.cc:2469] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:04.716436  1975 consensus_queue.cc:1049] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:04.718713  1046 tablet_service.cc:1495] Processing CreateTablet for tablet c9fa78b08d5c4a0f95a052e429ca7f2a (DEFAULT_TABLE table=default.Table_4cf0ca76843b4dc3982bdaee328cb161 [id=592e1a71bd7e4aedac82ab20be6c245d]), partition=RANGE (key) PARTITION UNBOUNDED
W20261120 03:01:04.719388  1011 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.96.65:39039, user_credentials={real_user=slave}} blocked reactor thread for 49155.3us
I20261120 03:01:04.719969  1046 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c9fa78b08d5c4a0f95a052e429ca7f2a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:04.740286  1181 tablet_bootstrap.cc:654] T 4da5eb2f2f114cc792488c5c74bb1414 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:04.718364  1180 consensus_queue.cc:261] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
W20261120 03:01:04.745191  1012 leader_election.cc:343] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
W20261120 03:01:04.711798   494 rpcz_store.cc:267] Call kudu.master.MasterService.TSHeartbeat from 127.0.96.65:40295 (request call id 38) took 1103 ms (client timeout 1000 ms). Trace:
W20261120 03:01:04.759399   494 rpcz_store.cc:269] 1120 03:01:03.608315 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:03.623416 (+ 15101us) service_pool.cc:224] Handling call
1120 03:01:04.711692 (+1088276us) inbound_call.cc:177] Queueing success response
Metrics: {"mutex_wait_us":22943,"reported_tablets":38}
I20261120 03:01:04.763053  1181 tablet_bootstrap.cc:492] T 4da5eb2f2f114cc792488c5c74bb1414 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:04.763466  1181 ts_tablet_manager.cc:1449] T 4da5eb2f2f114cc792488c5c74bb1414 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.072s	user 0.009s	sys 0.002s
I20261120 03:01:04.765913  1059 tablet_service.cc:1495] Processing CreateTablet for tablet 4ebd4502672049d9ae09d4446e1a3447 (DEFAULT_TABLE table=default.Table_47101d55d8f6485c816a02a54fda67a8 [id=252d229ff1334cd99ec99e5c921031bb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:04.765955  1181 raft_consensus.cc:366] T 4da5eb2f2f114cc792488c5c74bb1414 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.766712  1181 raft_consensus.cc:392] T 4da5eb2f2f114cc792488c5c74bb1414 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:04.766969  1181 raft_consensus.cc:787] T 4da5eb2f2f114cc792488c5c74bb1414 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:04.767572  1181 consensus_queue.cc:261] T 4da5eb2f2f114cc792488c5c74bb1414 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.768114  1059 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4ebd4502672049d9ae09d4446e1a3447. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:04.769479  1181 ts_tablet_manager.cc:1480] T 4da5eb2f2f114cc792488c5c74bb1414 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.006s	user 0.004s	sys 0.000s
I20261120 03:01:04.770340  1181 tablet_bootstrap.cc:492] T 29e6a50c9e1f4b778737abdef9ad073a P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:04.771664  1915 ts_tablet_manager.cc:1975] T 1faba22e8e38418383b86e42e904a761 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:04.773393  1915 log.cc:1199] T 1faba22e8e38418383b86e42e904a761 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/1faba22e8e38418383b86e42e904a761
I20261120 03:01:04.780282  1915 ts_tablet_manager.cc:1996] T 1faba22e8e38418383b86e42e904a761 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:01:04.787537  1181 tablet_bootstrap.cc:654] T 29e6a50c9e1f4b778737abdef9ad073a P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
W20261120 03:01:04.794265   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 87717.8us
I20261120 03:01:04.794385   903 tablet_service.cc:1495] Processing CreateTablet for tablet c9fa78b08d5c4a0f95a052e429ca7f2a (DEFAULT_TABLE table=default.Table_4cf0ca76843b4dc3982bdaee328cb161 [id=592e1a71bd7e4aedac82ab20be6c245d]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:04.795804   903 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet c9fa78b08d5c4a0f95a052e429ca7f2a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:04.796046   900 tablet_service.cc:1495] Processing CreateTablet for tablet 4ebd4502672049d9ae09d4446e1a3447 (DEFAULT_TABLE table=default.Table_47101d55d8f6485c816a02a54fda67a8 [id=252d229ff1334cd99ec99e5c921031bb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:04.796977   900 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4ebd4502672049d9ae09d4446e1a3447. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:04.798031  1918 ts_tablet_manager.cc:1962] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:04.800271  1046 tablet_service.cc:1495] Processing CreateTablet for tablet 2936e79caca943258b39849505c59f50 (DEFAULT_TABLE table=default.Table_9fe8b8c914534e4ca5602de7c1b1d280 [id=f3b5f7a4e5494e82a472d0025df96e0c]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:04.806420   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 1faba22e8e38418383b86e42e904a761 (table default.Table_47101d55d8f6485c816a02a54fda67a8 [id=252d229ff1334cd99ec99e5c921031bb]) successfully deleted
W20261120 03:01:04.797401  1996 log.cc:927] Time spent T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440: Append to log took a long time: real 0.083s	user 0.000s	sys 0.000s
I20261120 03:01:04.812480  1046 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2936e79caca943258b39849505c59f50. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:04.818544  1915 tablet_replica.cc:333] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:01:04.819286  1181 tablet_bootstrap.cc:492] T 29e6a50c9e1f4b778737abdef9ad073a P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:04.819691  1181 ts_tablet_manager.cc:1449] T 29e6a50c9e1f4b778737abdef9ad073a P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.050s	user 0.009s	sys 0.000s
I20261120 03:01:04.819661  1915 raft_consensus.cc:2440] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:01:04.820633  1915 raft_consensus.cc:2469] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:04.822132  1181 raft_consensus.cc:366] T 29e6a50c9e1f4b778737abdef9ad073a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.822731  1181 raft_consensus.cc:392] T 29e6a50c9e1f4b778737abdef9ad073a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
W20261120 03:01:04.823988   860 consensus_peers.cc:183] SignalRequest failed: Illegal state: peer closed
I20261120 03:01:04.825322  1051 tablet_service.cc:1495] Processing CreateTablet for tablet 17863073c05343a29aa0c4039ce7fa75 (DEFAULT_TABLE table=default.Table_e53618d9c5b7456f9e23d36a5f55affe [id=be850d2dfeaf4ba1a7f69b5447114a70]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:04.826431  1051 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 17863073c05343a29aa0c4039ce7fa75. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:04.834901   897 tablet_service.cc:1495] Processing CreateTablet for tablet 17863073c05343a29aa0c4039ce7fa75 (DEFAULT_TABLE table=default.Table_e53618d9c5b7456f9e23d36a5f55affe [id=be850d2dfeaf4ba1a7f69b5447114a70]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:04.836071   897 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 17863073c05343a29aa0c4039ce7fa75. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:04.798308   895 tablet_service.cc:1495] Processing CreateTablet for tablet 2936e79caca943258b39849505c59f50 (DEFAULT_TABLE table=default.Table_9fe8b8c914534e4ca5602de7c1b1d280 [id=f3b5f7a4e5494e82a472d0025df96e0c]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:04.838406   902 tablet_service.cc:1495] Processing CreateTablet for tablet 32373a40d0534dbd8ae530daaf4a5e00 (DEFAULT_TABLE table=default.Table_d1dabef467c84c168eaedfda4f61a259 [id=ae8045b03e974d6ab6d90be467e41cdc]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:04.838946   895 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 2936e79caca943258b39849505c59f50. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:04.839520   902 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 32373a40d0534dbd8ae530daaf4a5e00. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:04.859716   500 catalog_manager.cc:5856] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 reported cstate change: term changed from 0 to 1, leader changed from <none> to 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67). New cstate: current_term: 1 leader_uuid: "823abdbd90ba47bbacad0f8dcff78234" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: UNKNOWN } } }
I20261120 03:01:04.861572   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 5da3d2724dd34aed88e6b08983188004 (table default.Table_d2bd3060da1545548f422436c9c7c22e [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]). Expected version 2 got 0
I20261120 03:01:04.862241   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 0179a59749a14168a840e9d4e610be1f (table default.Table_cc56bf0ad8344f1a94d5da6e4d77e8c0 [id=b9034a083c7e4133a746b336335d3e42]). Expected version 1 got 0
I20261120 03:01:04.865146  1045 tablet_service.cc:1495] Processing CreateTablet for tablet 32373a40d0534dbd8ae530daaf4a5e00 (DEFAULT_TABLE table=default.Table_d1dabef467c84c168eaedfda4f61a259 [id=ae8045b03e974d6ab6d90be467e41cdc]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:04.866319  1045 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 32373a40d0534dbd8ae530daaf4a5e00. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:04.822979  1181 raft_consensus.cc:787] T 29e6a50c9e1f4b778737abdef9ad073a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:04.899878  1181 consensus_queue.cc:261] T 29e6a50c9e1f4b778737abdef9ad073a P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.901801  1181 ts_tablet_manager.cc:1480] T 29e6a50c9e1f4b778737abdef9ad073a P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.082s	user 0.006s	sys 0.000s
I20261120 03:01:04.902668  1181 tablet_bootstrap.cc:492] T 0dca5a2820ad4197b83a804b0310e57c P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
W20261120 03:01:04.908087   857 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.67:34451, user_credentials={real_user=slave}} blocked reactor thread for 113218us
W20261120 03:01:04.909678   858 consensus_peers.cc:183] SignalRequest failed: Illegal state: peer closed
W20261120 03:01:04.918937  1012 consensus_peers.cc:597] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 -> Peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Couldn't send request to peer 3d30c2428730481db9b0ffd279dd3440. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b549a06d15db449b851fe9549a963416. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:01:04.924835  1915 ts_tablet_manager.cc:1962] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:04.928282  1181 tablet_bootstrap.cc:654] T 0dca5a2820ad4197b83a804b0310e57c P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:04.930941  1068 raft_consensus.cc:3296] T 9dadc05b726346309d6b350f9abb4abb P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:04.941769  1076 raft_consensus.cc:3296] T 59cc37f7ef9c409493bd89d0d2097339 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:04.934203  1837 raft_consensus.cc:534] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:04.955765  1837 raft_consensus.cc:556] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:04.957652  1837 leader_election.cc:290] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:04.964105  1918 ts_tablet_manager.cc:1975] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:04.964493  1918 log.cc:1199] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/b3f7103b59d443f9bc739083b8e023ce
I20261120 03:01:04.965215  1918 ts_tablet_manager.cc:1996] T b3f7103b59d443f9bc739083b8e023ce P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:01:04.977494  1180 ts_tablet_manager.cc:1480] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.432s	user 0.027s	sys 0.050s
I20261120 03:01:04.978469  1180 tablet_bootstrap.cc:492] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:01:04.988334  1181 tablet_bootstrap.cc:492] T 0dca5a2820ad4197b83a804b0310e57c P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:04.988756  1181 ts_tablet_manager.cc:1449] T 0dca5a2820ad4197b83a804b0310e57c P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.086s	user 0.011s	sys 0.000s
I20261120 03:01:04.991091  1181 raft_consensus.cc:366] T 0dca5a2820ad4197b83a804b0310e57c P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.991732  1181 raft_consensus.cc:392] T 0dca5a2820ad4197b83a804b0310e57c P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:04.991982  1181 raft_consensus.cc:787] T 0dca5a2820ad4197b83a804b0310e57c P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:04.992564  1181 consensus_queue.cc:261] T 0dca5a2820ad4197b83a804b0310e57c P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:04.994447  1181 ts_tablet_manager.cc:1480] T 0dca5a2820ad4197b83a804b0310e57c P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.005s	user 0.007s	sys 0.000s
I20261120 03:01:04.995329  1181 tablet_bootstrap.cc:492] T bed7910ab7c140199de04e306c532969 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:05.000797   857 leader_election.cc:304] T ad49c8e00aca4a88a091baceae320477 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:05.001075  1180 tablet_bootstrap.cc:654] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:05.001855   500 catalog_manager.cc:5856] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 reported cstate change: term changed from 0 to 1, leader changed from <none> to 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67). New cstate: current_term: 1 leader_uuid: "823abdbd90ba47bbacad0f8dcff78234" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: UNKNOWN } } }
I20261120 03:01:05.003140   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 73c993944b0a4977b55d40ad63bb69e3 (table default.Table_a96b0ac352c545da9b955502704e266b [id=1cdd4bbccdbf4449ba67b9aa7089a6fd]). Expected version 1 got 0
I20261120 03:01:05.003198  1893 raft_consensus.cc:534] T 26f8ba69516c447dbd91e0d87cd519d3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:05.003752  1893 raft_consensus.cc:556] T 26f8ba69516c447dbd91e0d87cd519d3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:05.003862   500 catalog_manager.cc:5856] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 reported cstate change: term changed from 0 to 1, leader changed from <none> to 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67). New cstate: current_term: 1 leader_uuid: "823abdbd90ba47bbacad0f8dcff78234" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: UNKNOWN } } }
I20261120 03:01:05.007275  1893 leader_election.cc:290] T 26f8ba69516c447dbd91e0d87cd519d3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:05.001529  2016 raft_consensus.cc:2966] T ad49c8e00aca4a88a091baceae320477 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:05.003122   857 leader_election.cc:304] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:05.013219  2016 raft_consensus.cc:2966] T 00f7c7d95bae4d55be6b437701989ba3 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20261120 03:01:05.014510   857 leader_election.cc:304] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440; no voters: 5beeefe8bef54caeac463d05d8c30fa6, 823abdbd90ba47bbacad0f8dcff78234
I20261120 03:01:05.009202   924 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "26f8ba69516c447dbd91e0d87cd519d3" candidate_uuid: "823abdbd90ba47bbacad0f8dcff78234" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d30c2428730481db9b0ffd279dd3440" is_pre_election: true
I20261120 03:01:05.018919  2016 raft_consensus.cc:2966] T ba85dab0d238487181ce5c1592a2c7fd P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
W20261120 03:01:05.019227  1012 leader_election.cc:343] T 26f8ba69516c447dbd91e0d87cd519d3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:05.023068   500 catalog_manager.cc:5856] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 reported cstate change: term changed from 0 to 1, leader changed from <none> to 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67). New cstate: current_term: 1 leader_uuid: "823abdbd90ba47bbacad0f8dcff78234" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } health_report { overall_health: UNKNOWN } } }
I20261120 03:01:05.029584  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9e42a416be884341994d2d5395286798" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:05.030222  1181 tablet_bootstrap.cc:654] T bed7910ab7c140199de04e306c532969 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:05.034942  1180 tablet_bootstrap.cc:492] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:01:05.035356  1180 ts_tablet_manager.cc:1449] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.057s	user 0.008s	sys 0.001s
I20261120 03:01:05.036365  1915 ts_tablet_manager.cc:1975] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:05.036755  1915 log.cc:1199] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/b3f7103b59d443f9bc739083b8e023ce
I20261120 03:01:05.037449  1915 ts_tablet_manager.cc:1996] T b3f7103b59d443f9bc739083b8e023ce P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:01:05.039598  1074 raft_consensus.cc:2062] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:01:05.037678  1180 raft_consensus.cc:366] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:05.042320  1180 raft_consensus.cc:392] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:05.042574  1180 raft_consensus.cc:787] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:05.043294  1180 consensus_queue.cc:261] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:05.045507  1180 ts_tablet_manager.cc:1480] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.010s	user 0.003s	sys 0.003s
I20261120 03:01:05.049760   495 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 73c993944b0a4977b55d40ad63bb69e3 (table default.Table_a96b0ac352c545da9b955502704e266b [id=1cdd4bbccdbf4449ba67b9aa7089a6fd]). Expected version 1 got 0
I20261120 03:01:05.049798   857 leader_election.cc:304] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:01:05.050491  2016 raft_consensus.cc:3024] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:05.050843  2016 raft_consensus.cc:534] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:05.051136  2016 raft_consensus.cc:3296] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:05.051900  2016 raft_consensus.cc:556] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:05.054558  1180 tablet_bootstrap.cc:492] T 4da5eb2f2f114cc792488c5c74bb1414 P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:01:05.060760  2016 leader_election.cc:290] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:05.068459  1180 tablet_bootstrap.cc:654] T 4da5eb2f2f114cc792488c5c74bb1414 P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:05.083968  1074 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9e42a416be884341994d2d5395286798" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:01:05.084564  1074 raft_consensus.cc:3296] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:05.093788  1915 tablet_replica.cc:333] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:01:05.093971   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet b3f7103b59d443f9bc739083b8e023ce (table default.Table_555e4ecfadce486baefa8c3e2d1bc272 [id=def72d7bfc214aa5a942c18714c0173e]) successfully deleted
W20261120 03:01:05.094707   486 catalog_manager.cc:5048] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): delete failed for tablet 0db7db25d779459ebde532f938493975 because the tablet was not found. No further retry: Not found: Tablet not found: 0db7db25d779459ebde532f938493975
I20261120 03:01:05.094756  1915 raft_consensus.cc:2440] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:05.095220  1915 raft_consensus.cc:2469] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:05.095803   500 catalog_manager.cc:6409] T 00000000000000000000000000000000 P 18331bef5c884dcab9dbd26bc2c82273: default.Table_12409ca618284aa285f193bbe6547101 [id=376c8b210df847dda73d22051971b065] alter complete (version 1)
I20261120 03:01:05.097260  1915 ts_tablet_manager.cc:1962] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:05.098430  1074 raft_consensus.cc:2062] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:01:05.116277  1046 tablet_service.cc:1542] Processing DeleteTablet for tablet 9e42a416be884341994d2d5395286798 with delete_type TABLET_DATA_DELETED (replaced by tablet ac349a09a0524b2d87289c3dd53adf6a) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:05.127611   857 leader_election.cc:304] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:01:05.136263  2013 raft_consensus.cc:3024] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:05.138161  1180 tablet_bootstrap.cc:492] T 4da5eb2f2f114cc792488c5c74bb1414 P 3d30c2428730481db9b0ffd279dd3440: No bootstrap required, opened a new log
I20261120 03:01:05.138547  1180 ts_tablet_manager.cc:1449] T 4da5eb2f2f114cc792488c5c74bb1414 P 3d30c2428730481db9b0ffd279dd3440: Time spent bootstrapping tablet: real 0.092s	user 0.021s	sys 0.002s
I20261120 03:01:05.138029  2013 raft_consensus.cc:744] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:01:05.139317  2013 consensus_queue.cc:238] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:05.149559   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet b3f7103b59d443f9bc739083b8e023ce (table default.Table_555e4ecfadce486baefa8c3e2d1bc272 [id=def72d7bfc214aa5a942c18714c0173e]) successfully deleted
W20261120 03:01:05.150297   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet b549a06d15db449b851fe9549a963416 because the tablet was not found. No further retry: Not found: Tablet not found: b549a06d15db449b851fe9549a963416
W20261120 03:01:05.150892   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet c6a65c1a4a574961bec657ad7c9048e0 because the tablet was not found. No further retry: Not found: Tablet not found: c6a65c1a4a574961bec657ad7c9048e0
I20261120 03:01:05.151108  2001 tablet.cc:1867] T b6f0ff2246cd4772af47812b699e93e8 P 823abdbd90ba47bbacad0f8dcff78234: Skipping requested alter to schema version 2, tablet already version 2
W20261120 03:01:05.151418   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 0db7db25d779459ebde532f938493975 because the tablet was not found. No further retry: Not found: Tablet not found: 0db7db25d779459ebde532f938493975
W20261120 03:01:05.151935   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet a1192a7f050a4f5387a30b2216dec5f0 because the tablet was not found. No further retry: Not found: Tablet not found: a1192a7f050a4f5387a30b2216dec5f0
W20261120 03:01:05.152450   487 catalog_manager.cc:5048] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): delete failed for tablet 1faba22e8e38418383b86e42e904a761 because the tablet was not found. No further retry: Not found: Tablet not found: 1faba22e8e38418383b86e42e904a761
I20261120 03:01:05.154104   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 73c993944b0a4977b55d40ad63bb69e3 (table default.Table_a96b0ac352c545da9b955502704e266b [id=1cdd4bbccdbf4449ba67b9aa7089a6fd]). Expected version 1 got 0
I20261120 03:01:05.135219   895 tablet_service.cc:1542] Processing DeleteTablet for tablet 9e42a416be884341994d2d5395286798 with delete_type TABLET_DATA_DELETED (replaced by tablet ac349a09a0524b2d87289c3dd53adf6a) from {username='slave'} at 127.0.0.1:44896
I20261120 03:01:05.155013   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 0179a59749a14168a840e9d4e610be1f (table default.Table_cc56bf0ad8344f1a94d5da6e4d77e8c0 [id=b9034a083c7e4133a746b336335d3e42]). Expected version 1 got 0
I20261120 03:01:05.158365  1918 tablet_replica.cc:333] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440: stopping tablet replica
I20261120 03:01:05.171437   495 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 73c993944b0a4977b55d40ad63bb69e3 (table default.Table_a96b0ac352c545da9b955502704e266b [id=1cdd4bbccdbf4449ba67b9aa7089a6fd]). Expected version 1 got 0
I20261120 03:01:05.171590  2028 tablet.cc:1867] T b6f0ff2246cd4772af47812b699e93e8 P 3d30c2428730481db9b0ffd279dd3440: Skipping requested alter to schema version 2, tablet already version 2
I20261120 03:01:05.172624   495 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 0179a59749a14168a840e9d4e610be1f (table default.Table_cc56bf0ad8344f1a94d5da6e4d77e8c0 [id=b9034a083c7e4133a746b336335d3e42]). Expected version 1 got 0
I20261120 03:01:05.173295   495 catalog_manager.cc:5956] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987) does not have the latest schema for tablet 5da3d2724dd34aed88e6b08983188004 (table default.Table_d2bd3060da1545548f422436c9c7c22e [id=f5d860e17cd74d6aa7b08fbbb0f8c75a]). Expected version 2 got 0
I20261120 03:01:05.180879  1918 raft_consensus.cc:2440] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:01:05.183161  1046 tablet_service.cc:1542] Processing DeleteTablet for tablet 29e6a50c9e1f4b778737abdef9ad073a with delete_type TABLET_DATA_DELETED (replaced by tablet 81d7cb5526944095b7054464971e8fa4) from {username='slave'} at 127.0.0.1:50744
I20261120 03:01:05.193840  1918 pending_rounds.cc:70] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440: Trying to abort 1 pending ops.
I20261120 03:01:05.194130  1918 pending_rounds.cc:77] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440: Aborting op as it isn't in flight: id { term: 1 index: 1 } timestamp: 7352908452391944192 op_type: NO_OP noop_request { }
I20261120 03:01:05.194526  1918 raft_consensus.cc:3115] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: NO_OP replication failed: Aborted: Op aborted
I20261120 03:01:05.194866  1918 raft_consensus.cc:2469] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:05.199281  1181 tablet_bootstrap.cc:492] T bed7910ab7c140199de04e306c532969 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:05.208316  1180 raft_consensus.cc:366] T 4da5eb2f2f114cc792488c5c74bb1414 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:05.208963  1180 raft_consensus.cc:392] T 4da5eb2f2f114cc792488c5c74bb1414 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:05.209214  1180 raft_consensus.cc:787] T 4da5eb2f2f114cc792488c5c74bb1414 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Initialized, Role: FOLLOWER
I20261120 03:01:05.209825  1181 ts_tablet_manager.cc:1449] T bed7910ab7c140199de04e306c532969 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.215s	user 0.035s	sys 0.000s
I20261120 03:01:05.212292  1181 raft_consensus.cc:366] T bed7910ab7c140199de04e306c532969 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:05.212911  1181 raft_consensus.cc:392] T bed7910ab7c140199de04e306c532969 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:05.213158  1181 raft_consensus.cc:787] T bed7910ab7c140199de04e306c532969 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:05.220971  2028 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.AlterSchema from 127.0.0.1:44896 (request call id 164) took 2349 ms. Trace:
I20261120 03:01:05.221375  2028 rpcz_store.cc:276] 1120 03:01:02.871097 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:02.871214 (+   117us) service_pool.cc:224] Handling call
1120 03:01:05.220794 (+2349580us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:02.873646 (+     0us) alter_schema_op.cc:104] PREPARE ALTER-SCHEMA: Starting
1120 03:01:02.874159 (+   513us) alter_schema_op.cc:63] Acquiring schema lock in exclusive mode
1120 03:01:03.261504 (+387345us) alter_schema_op.cc:66] Acquired schema lock
1120 03:01:03.261586 (+    82us) alter_schema_op.cc:119] PREPARE ALTER-SCHEMA: finished
1120 03:01:03.261708 (+   122us) alter_schema_op.cc:128] START. Timestamp: 1795143663261640
1120 03:01:03.261729 (+    21us) op_driver.cc:350] REPLICATION: starting
1120 03:01:03.262258 (+   529us) log.cc:844] Serialized 173 byte log entry
1120 03:01:05.137536 (+1875278us) op_driver.cc:466] REPLICATION: finished
1120 03:01:05.171293 (+ 33757us) alter_schema_op.cc:133] APPLY ALTER-SCHEMA: Starting
1120 03:01:05.172326 (+  1033us) log.cc:844] Serialized 94 byte log entry
1120 03:01:05.220649 (+ 48323us) alter_schema_op.cc:73] Released schema lock
1120 03:01:05.220679 (+    30us) alter_schema_op.cc:186] AlterSchemaCommitCallback: making alter schema visible
Metrics: {"child_traces":[["op",{"apply.queue_time_us":33485,"prepare.queue_time_us":1527,"prepare.run_cpu_time_us":15781,"prepare.run_wall_time_us":389730,"replication_time_us":1875670,"thread_start_us":83335,"threads_started":2,"wal-append.queue_time_us":47090}]]}
I20261120 03:01:05.225939  1918 ts_tablet_manager.cc:1962] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:05.222381  1180 consensus_queue.cc:261] T 4da5eb2f2f114cc792488c5c74bb1414 P 3d30c2428730481db9b0ffd279dd3440 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:05.237938  1180 ts_tablet_manager.cc:1480] T 4da5eb2f2f114cc792488c5c74bb1414 P 3d30c2428730481db9b0ffd279dd3440: Time spent starting tablet: real 0.032s	user 0.004s	sys 0.001s
I20261120 03:01:05.213742  1181 consensus_queue.cc:261] T bed7910ab7c140199de04e306c532969 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:05.252708  1180 tablet_bootstrap.cc:492] T 29e6a50c9e1f4b778737abdef9ad073a P 3d30c2428730481db9b0ffd279dd3440: Bootstrap starting.
I20261120 03:01:05.263767  1915 ts_tablet_manager.cc:1975] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:05.293740  2013 raft_consensus.cc:534] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:05.294241  2013 raft_consensus.cc:556] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:05.296118  2013 leader_election.cc:290] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:05.296726  2013 raft_consensus.cc:534] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:05.297940  1915 log.cc:1199] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/014bd9c83be34c31b90fe4bcf31f1649
I20261120 03:01:05.297116  2013 raft_consensus.cc:556] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:05.299875  2013 leader_election.cc:290] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:05.267058  1181 ts_tablet_manager.cc:1480] T bed7910ab7c140199de04e306c532969 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.057s	user 0.005s	sys 0.000s
I20261120 03:01:05.300789  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "600a18b7e5104a78935d906b8adfde6f" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:05.301472  1075 raft_consensus.cc:2062] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:01:05.297281  1076 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5da3d2724dd34aed88e6b08983188004" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234" is_pre_election: true
I20261120 03:01:05.302698  1076 raft_consensus.cc:2062] T 5da3d2724dd34aed88e6b08983188004 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 0
I20261120 03:01:05.303875   857 leader_election.cc:304] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:01:05.304737   857 leader_election.cc:304] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:01:05.305425  2016 raft_consensus.cc:3024] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader pre-election won for term 2
I20261120 03:01:05.305470  2013 raft_consensus.cc:3024] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:05.305806  2016 raft_consensus.cc:534] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:05.305882  2013 raft_consensus.cc:534] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:05.306166  2016 raft_consensus.cc:3296] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Advancing to term 2
I20261120 03:01:05.306344  2013 raft_consensus.cc:3296] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:05.306950  2016 raft_consensus.cc:556] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } }
I20261120 03:01:05.313774  2013 raft_consensus.cc:556] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:05.314736  2016 leader_election.cc:290] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 2 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:05.316296  1181 tablet_bootstrap.cc:492] T 0c36259ff308455d986243c6b07d81a6 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:05.318210  1915 ts_tablet_manager.cc:1996] T 014bd9c83be34c31b90fe4bcf31f1649 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
I20261120 03:01:05.321605  1918 ts_tablet_manager.cc:1975] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:05.322029  1181 tablet_bootstrap.cc:654] T 0c36259ff308455d986243c6b07d81a6 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:05.339943  1076 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "600a18b7e5104a78935d906b8adfde6f" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:01:05.340564  1076 raft_consensus.cc:3296] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Advancing to term 2
I20261120 03:01:05.341439  1076 raft_consensus.cc:2062] T 600a18b7e5104a78935d906b8adfde6f P 823abdbd90ba47bbacad0f8dcff78234 [term 2 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 2
I20261120 03:01:05.342511  1181 tablet_bootstrap.cc:492] T 0c36259ff308455d986243c6b07d81a6 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:05.342911  1181 ts_tablet_manager.cc:1449] T 0c36259ff308455d986243c6b07d81a6 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.027s	user 0.013s	sys 0.000s
I20261120 03:01:05.343338  1075 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "5da3d2724dd34aed88e6b08983188004" candidate_uuid: "3d30c2428730481db9b0ffd279dd3440" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "823abdbd90ba47bbacad0f8dcff78234"
I20261120 03:01:05.343834  1075 raft_consensus.cc:3296] T 5da3d2724dd34aed88e6b08983188004 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:05.344705  1075 raft_consensus.cc:2062] T 5da3d2724dd34aed88e6b08983188004 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 3d30c2428730481db9b0ffd279dd3440 in term 1
I20261120 03:01:05.346840  1181 raft_consensus.cc:366] T 0c36259ff308455d986243c6b07d81a6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:05.347440  1181 raft_consensus.cc:392] T 0c36259ff308455d986243c6b07d81a6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:05.347684  1181 raft_consensus.cc:787] T 0c36259ff308455d986243c6b07d81a6 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:05.348678  1181 consensus_queue.cc:261] T 0c36259ff308455d986243c6b07d81a6 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:05.350764  1181 ts_tablet_manager.cc:1480] T 0c36259ff308455d986243c6b07d81a6 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.008s	user 0.005s	sys 0.001s
I20261120 03:01:05.355988  1181 tablet_bootstrap.cc:492] T 8935b23c661c4c1ca7bde7c893927502 P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
W20261120 03:01:05.357748  1641 connection.cc:573] server connection from 127.0.96.65:49637 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
I20261120 03:01:05.361241  1181 tablet_bootstrap.cc:654] T 8935b23c661c4c1ca7bde7c893927502 P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:05.323275  2013 leader_election.cc:290] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Requested vote from peers 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039), 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451)
I20261120 03:01:05.323822  1180 tablet_bootstrap.cc:654] T 29e6a50c9e1f4b778737abdef9ad073a P 3d30c2428730481db9b0ffd279dd3440: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:05.363214  1918 log.cc:1199] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/9e42a416be884341994d2d5395286798
I20261120 03:01:05.363986  1918 ts_tablet_manager.cc:1996] T 9e42a416be884341994d2d5395286798 P 3d30c2428730481db9b0ffd279dd3440: Deleting consensus metadata
I20261120 03:01:05.365978   487 catalog_manager.cc:5081] TS 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987): tablet 9e42a416be884341994d2d5395286798 (table default.Table_e4c5ff8f28ae4efca67257942a4fcb77 [id=b30488f6d14b4c9ab1566f2376be602a]) successfully deleted
I20261120 03:01:05.368250   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet 014bd9c83be34c31b90fe4bcf31f1649 (table default.Table_cfa8989fb1134d908ab28bd6ebb80563 [id=16bb3f7e35c54a8c936b4af5a66a7f17]) successfully deleted
I20261120 03:01:05.373077  1915 tablet_replica.cc:333] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:01:05.362399   857 leader_election.cc:304] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:01:05.375257  1967 raft_consensus.cc:3024] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440 [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:05.375697  1967 raft_consensus.cc:744] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440 [term 1 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:01:05.376463  1967 consensus_queue.cc:238] T 5da3d2724dd34aed88e6b08983188004 P 3d30c2428730481db9b0ffd279dd3440 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:05.377291  1915 raft_consensus.cc:2440] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:01:05.379153  1915 raft_consensus.cc:2469] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
W20261120 03:01:05.380499   486 connection.cc:573] client connection to 127.0.96.65:39039 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:05.380465   411 connection.cc:573] server connection from 127.0.96.65:42803 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:05.380856   486 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:05.381450   859 connection.cc:573] server connection from 127.0.96.65:51153 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:05.382213   489 connection.cc:573] server connection from 127.0.96.65:40295 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:05.379699  1011 connection.cc:573] client connection to 127.0.96.65:39039 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:05.389518  1011 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Connection reset by peer (error 104)
I20261120 03:01:05.397418   857 leader_election.cc:304] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3d30c2428730481db9b0ffd279dd3440, 823abdbd90ba47bbacad0f8dcff78234; no voters: 
I20261120 03:01:05.400269  1915 ts_tablet_manager.cc:1962] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:05.400398  2013 raft_consensus.cc:3024] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 2 FOLLOWER]: Leader election won for term 2
I20261120 03:01:05.400859  2013 raft_consensus.cc:744] T 600a18b7e5104a78935d906b8adfde6f P 3d30c2428730481db9b0ffd279dd3440 [term 2 LEADER]: Becoming Leader. State: Replica: 3d30c2428730481db9b0ffd279dd3440, State: Running, Role: LEADER
I20261120 03:01:05.401002   385 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu with pid 830
I20261120 03:01:05.418047  1181 tablet_bootstrap.cc:492] T 8935b23c661c4c1ca7bde7c893927502 P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:05.418520  1181 ts_tablet_manager.cc:1449] T 8935b23c661c4c1ca7bde7c893927502 P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.063s	user 0.011s	sys 0.004s
I20261120 03:01:05.421060  1181 raft_consensus.cc:366] T 8935b23c661c4c1ca7bde7c893927502 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:05.421684  1181 raft_consensus.cc:392] T 8935b23c661c4c1ca7bde7c893927502 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:05.421944  1181 raft_consensus.cc:787] T 8935b23c661c4c1ca7bde7c893927502 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:05.422605  1181 consensus_queue.cc:261] T 8935b23c661c4c1ca7bde7c893927502 P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:05.424618  1181 ts_tablet_manager.cc:1480] T 8935b23c661c4c1ca7bde7c893927502 P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.006s	user 0.003s	sys 0.000s
W20261120 03:01:05.429683   486 catalog_manager.cc:4808] TS 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): DeleteTablet:TABLET_DATA_DELETED RPC failed for tablet 5c5c0ac869a1419fad9f61ec647027ee: Network error: Client connection negotiation failed: client connection to 127.0.96.65:39039: connect: Connection refused (error 111)
I20261120 03:01:05.434976  1181 tablet_bootstrap.cc:492] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:05.440517  1181 tablet_bootstrap.cc:654] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:05.441809  1972 raft_consensus.cc:534] T 8935b23c661c4c1ca7bde7c893927502 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:05.442301  1972 raft_consensus.cc:556] T 8935b23c661c4c1ca7bde7c893927502 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:05.443276   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 8935b23c661c4c1ca7bde7c893927502 (table default.Table_36247824df9547f8b6c7cb6ca56fb0ea [id=51df6e371d3441578e57ea5597ee81a6]). Expected version 2 got 0
I20261120 03:01:05.444272  1972 leader_election.cc:290] T 8935b23c661c4c1ca7bde7c893927502 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:05.458526   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 8935b23c661c4c1ca7bde7c893927502 (table default.Table_36247824df9547f8b6c7cb6ca56fb0ea [id=51df6e371d3441578e57ea5597ee81a6]). Expected version 2 got 0
W20261120 03:01:05.458959  1011 leader_election.cc:336] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Network error: Client connection negotiation failed: client connection to 127.0.96.65:39039: connect: Connection refused (error 111)
I20261120 03:01:05.459470  1011 leader_election.cc:304] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:01:05.461697  1972 raft_consensus.cc:2966] T 9ea5f0b680bd401bbc2b69d780e1ef50 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20261120 03:01:05.463050  1011 consensus_peers.cc:597] T ad49c8e00aca4a88a091baceae320477 P 823abdbd90ba47bbacad0f8dcff78234 -> Peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Couldn't send request to peer 5beeefe8bef54caeac463d05d8c30fa6. Status: Network error: Client connection negotiation failed: client connection to 127.0.96.65:39039: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20261120 03:01:05.470275  1011 consensus_peers.cc:597] T 73c993944b0a4977b55d40ad63bb69e3 P 823abdbd90ba47bbacad0f8dcff78234 -> Peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Couldn't send request to peer 5beeefe8bef54caeac463d05d8c30fa6. Status: Network error: Client connection negotiation failed: client connection to 127.0.96.65:39039: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20261120 03:01:05.471592  1011 consensus_peers.cc:597] T 00f7c7d95bae4d55be6b437701989ba3 P 823abdbd90ba47bbacad0f8dcff78234 -> Peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Couldn't send request to peer 5beeefe8bef54caeac463d05d8c30fa6. Status: Network error: Client connection negotiation failed: client connection to 127.0.96.65:39039: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20261120 03:01:05.472077  1011 consensus_peers.cc:597] T ba85dab0d238487181ce5c1592a2c7fd P 823abdbd90ba47bbacad0f8dcff78234 -> Peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Couldn't send request to peer 5beeefe8bef54caeac463d05d8c30fa6. Status: Network error: Client connection negotiation failed: client connection to 127.0.96.65:39039: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20261120 03:01:05.472508  1011 leader_election.cc:336] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Network error: Client connection negotiation failed: client connection to 127.0.96.65:39039: connect: Connection refused (error 111)
I20261120 03:01:05.472920  1011 leader_election.cc:304] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:01:05.473697  1972 raft_consensus.cc:2966] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20261120 03:01:05.474839  1011 consensus_peers.cc:597] T 33e6cd23a3474034953c9b1a37b79acd P 823abdbd90ba47bbacad0f8dcff78234 -> Peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Couldn't send request to peer 5beeefe8bef54caeac463d05d8c30fa6. Status: Network error: Client connection negotiation failed: client connection to 127.0.96.65:39039: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20261120 03:01:05.475696  1011 leader_election.cc:336] T 26f8ba69516c447dbd91e0d87cd519d3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Network error: Client connection negotiation failed: client connection to 127.0.96.65:39039: connect: Connection refused (error 111)
I20261120 03:01:05.479748  1011 leader_election.cc:304] T 26f8ba69516c447dbd91e0d87cd519d3 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 823abdbd90ba47bbacad0f8dcff78234; no voters: 3d30c2428730481db9b0ffd279dd3440, 5beeefe8bef54caeac463d05d8c30fa6
I20261120 03:01:05.480551  1972 raft_consensus.cc:2966] T 26f8ba69516c447dbd91e0d87cd519d3 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20261120 03:01:05.481839  1181 tablet_bootstrap.cc:492] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:05.482267  1181 ts_tablet_manager.cc:1449] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.047s	user 0.011s	sys 0.013s
I20261120 03:01:05.484877  1181 raft_consensus.cc:366] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:05.485512  1181 raft_consensus.cc:392] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:05.485769  1181 raft_consensus.cc:787] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:05.498210  1181 consensus_queue.cc:261] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:05.501151  1181 ts_tablet_manager.cc:1480] T 4d79fc4c8f1c4a5f83c0dc94337c857e P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.019s	user 0.001s	sys 0.003s
I20261120 03:01:05.502445  1181 tablet_bootstrap.cc:492] T 8f1fda857e6c400e80d3d177a3d748ff P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
W20261120 03:01:05.507445  1011 leader_election.cc:336] T 8935b23c661c4c1ca7bde7c893927502 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Network error: Client connection negotiation failed: client connection to 127.0.96.65:39039: connect: Connection refused (error 111)
I20261120 03:01:05.507970  1181 tablet_bootstrap.cc:654] T 8f1fda857e6c400e80d3d177a3d748ff P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:05.519255  1915 ts_tablet_manager.cc:1975] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:05.525173   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 4d79fc4c8f1c4a5f83c0dc94337c857e (table default.Table_650dc67e0ac74c149a3f64d456767938 [id=cfcd5fea97fb4ab3baff99eb28cd9476]). Expected version 3 got 0
W20261120 03:01:05.526667   500 catalog_manager.cc:6056] Failed to send AlterTable RPC for tablet 4d79fc4c8f1c4a5f83c0dc94337c857e (table default.Table_650dc67e0ac74c149a3f64d456767938 [id=cfcd5fea97fb4ab3baff99eb28cd9476], current schema version=3): Not found: failed to reset TS proxy: No leader found: Tablet 4d79fc4c8f1c4a5f83c0dc94337c857e consensus state has no leader
I20261120 03:01:05.522006  1972 raft_consensus.cc:534] T 29e6a50c9e1f4b778737abdef9ad073a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:05.523674  1181 tablet_bootstrap.cc:492] T 8f1fda857e6c400e80d3d177a3d748ff P 823abdbd90ba47bbacad0f8dcff78234: No bootstrap required, opened a new log
I20261120 03:01:05.539979  1893 raft_consensus.cc:534] T 4da5eb2f2f114cc792488c5c74bb1414 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:05.549686  1181 ts_tablet_manager.cc:1449] T 8f1fda857e6c400e80d3d177a3d748ff P 823abdbd90ba47bbacad0f8dcff78234: Time spent bootstrapping tablet: real 0.047s	user 0.012s	sys 0.001s
I20261120 03:01:05.549592  1972 raft_consensus.cc:556] T 29e6a50c9e1f4b778737abdef9ad073a P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:05.549678  1893 raft_consensus.cc:556] T 4da5eb2f2f114cc792488c5c74bb1414 P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } } peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } }
I20261120 03:01:05.551647  1972 leader_election.cc:290] T 29e6a50c9e1f4b778737abdef9ad073a P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:05.552181  1181 raft_consensus.cc:366] T 8f1fda857e6c400e80d3d177a3d748ff P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:05.552769  1181 raft_consensus.cc:392] T 8f1fda857e6c400e80d3d177a3d748ff P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:05.551647  1893 leader_election.cc:290] T 4da5eb2f2f114cc792488c5c74bb1414 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d30c2428730481db9b0ffd279dd3440 (127.0.96.66:42987), 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039)
I20261120 03:01:05.553009  1181 raft_consensus.cc:787] T 8f1fda857e6c400e80d3d177a3d748ff P 823abdbd90ba47bbacad0f8dcff78234 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 823abdbd90ba47bbacad0f8dcff78234, State: Initialized, Role: FOLLOWER
I20261120 03:01:05.553599  1181 consensus_queue.cc:261] T 8f1fda857e6c400e80d3d177a3d748ff P 823abdbd90ba47bbacad0f8dcff78234 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5beeefe8bef54caeac463d05d8c30fa6" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 39039 } } peers { permanent_uuid: "3d30c2428730481db9b0ffd279dd3440" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 42987 } } peers { permanent_uuid: "823abdbd90ba47bbacad0f8dcff78234" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 34451 } }
I20261120 03:01:05.555682  1181 ts_tablet_manager.cc:1480] T 8f1fda857e6c400e80d3d177a3d748ff P 823abdbd90ba47bbacad0f8dcff78234: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.000s
I20261120 03:01:05.557166   500 catalog_manager.cc:5956] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451) does not have the latest schema for tablet 4d79fc4c8f1c4a5f83c0dc94337c857e (table default.Table_650dc67e0ac74c149a3f64d456767938 [id=cfcd5fea97fb4ab3baff99eb28cd9476]). Expected version 3 got 0
I20261120 03:01:05.558279  1181 tablet_bootstrap.cc:492] T 5ea1646a2ab84345a77fd7ffd249003a P 823abdbd90ba47bbacad0f8dcff78234: Bootstrap starting.
I20261120 03:01:05.567708  1181 tablet_bootstrap.cc:654] T 5ea1646a2ab84345a77fd7ffd249003a P 823abdbd90ba47bbacad0f8dcff78234: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:05.568773  1915 log.cc:1199] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_0.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/b549a06d15db449b851fe9549a963416
W20261120 03:01:05.569147   486 catalog_manager.cc:4863] Async tablet task AlterTable RPC for tablet 4d79fc4c8f1c4a5f83c0dc94337c857e (table default.Table_650dc67e0ac74c149a3f64d456767938 [id=cfcd5fea97fb4ab3baff99eb28cd9476], current schema version=3) failed: Not found: failed to reset TS proxy: No leader found: Tablet 4d79fc4c8f1c4a5f83c0dc94337c857e consensus state has no leader
I20261120 03:01:05.578611  1915 ts_tablet_manager.cc:1996] T b549a06d15db449b851fe9549a963416 P 823abdbd90ba47bbacad0f8dcff78234: Deleting consensus metadata
W20261120 03:01:05.580763  1011 leader_election.cc:336] T 4da5eb2f2f114cc792488c5c74bb1414 P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Network error: Client connection negotiation failed: client connection to 127.0.96.65:39039: connect: Connection refused (error 111)
W20261120 03:01:05.584331  1011 leader_election.cc:336] T 29e6a50c9e1f4b778737abdef9ad073a P 823abdbd90ba47bbacad0f8dcff78234 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5beeefe8bef54caeac463d05d8c30fa6 (127.0.96.65:39039): Network error: Client connection negotiation failed: client connection to 127.0.96.65:39039: connect: Connection refused (error 111)
I20261120 03:01:05.598331  1915 tablet_replica.cc:333] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234: stopping tablet replica
I20261120 03:01:05.601445  1915 raft_consensus.cc:2440] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:05.603813   486 catalog_manager.cc:5081] TS 823abdbd90ba47bbacad0f8dcff78234 (127.0.96.67:34451): tablet b549a06d15db449b851fe9549a963416 (table default.Table_61cb1e03b7e442a8822e32a2295866f3 [id=b5b565c2c2224675baada2ae9b42be54]) successfully deleted
I20261120 03:01:05.609915  1915 raft_consensus.cc:2469] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234 [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:05.616580  1915 ts_tablet_manager.cc:1962] T 9e42a416be884341994d2d5395286798 P 823abdbd90ba47bbacad0f8dcff78234: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:05.629426   385 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu with pid 977
W20261120 03:01:05.715200   412 connection.cc:573] server connection from 127.0.96.67:52173 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
I20261120 03:01:05.715700   385 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu with pid 394
I20261120 03:01:05.759172   385 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu with pid 464
I20261120 03:01:05.809784   385 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu with pid 1300
2026-11-20T03:01:05Z chronyd exiting
I20261120 03:01:05.853802   385 master-stress-test.cc:587] Tables created: 96
I20261120 03:01:05.853993   385 master-stress-test.cc:588] Tables altered: 39
I20261120 03:01:05.854167   385 master-stress-test.cc:589] Tables deleted: 38
I20261120 03:01:05.854348   385 master-stress-test.cc:590] Tablets replaced: 17
I20261120 03:01:05.854468   385 master-stress-test.cc:591] Masters restarted: 1
[       OK ] CatalogManagerConfigurations/MasterStressTest.Test/0 (33141 ms)
[ RUN      ] CatalogManagerConfigurations/MasterStressTest.Test/3
2026-11-20T03:01:05Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
2026-11-20T03:01:05Z Disabled control of system clock
Picked up JAVA_TOOL_OPTIONS: -Djava.net.preferIPv4Stack=true -Dderby.locks.deadlockTimeout=1 -Dderby.stream.error.file=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/derby.log -Djava.security.properties=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/security.properties
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/tmp/dist-test-tasklxEiy3/thirdparty/installed/common/opt/hive/lib/log4j-slf4j-impl-2.10.0.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/tmp/dist-test-tasklxEiy3/thirdparty/installed/common/opt/hadoop/share/hadoop/common/lib/slf4j-reload4j-1.7.36.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
Metastore connection URL:	 jdbc:derby:/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/metadb;create=true
Metastore Connection Driver :	 org.apache.derby.jdbc.EmbeddedDriver
Metastore connection User:	 APP
Starting metastore schema initialization to 3.1.0
Initialization script hive-schema-3.1.0.derby.sql

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 

 
 

 
 
 
 

 
 
 
 


 
 
 

 
 
 


 
 
 
 
 
 
 


 
 
 

 
 
 

 
 

 
 
 
 
 

 
 
 
 
 
 
 
 
 
 
 
 

 
 
 
 
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 

 
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 

 
 
 
 


 
 
 
 
 
 



 
 
 
 
 


 
 
 
 
 
 
 
 
 
 
 
 
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 

 


Initialization script completed
schemaTool completed
Picked up JAVA_TOOL_OPTIONS: -Djava.net.preferIPv4Stack=true -Dderby.locks.deadlockTimeout=1 -Dderby.stream.error.file=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/derby.log -Djava.security.properties=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/security.properties
2026-11-20 03:01:12: Starting Hive Metastore Server
Picked up JAVA_TOOL_OPTIONS: -Djava.net.preferIPv4Stack=true -Dderby.locks.deadlockTimeout=1 -Dderby.stream.error.file=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/derby.log -Djava.security.properties=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/security.properties
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/tmp/dist-test-tasklxEiy3/thirdparty/installed/common/opt/hive/lib/log4j-slf4j-impl-2.10.0.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/tmp/dist-test-tasklxEiy3/thirdparty/installed/common/opt/hadoop/share/hadoop/common/lib/slf4j-reload4j-1.7.36.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
Starting hive metastore on port 0
I20261120 03:01:22.033795   385 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
/tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/wal
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/logs
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.126:46565
--webserver_interface=127.0.96.126
--webserver_port=0
--builtin_ntp_servers=127.0.96.84:36533
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751
--hive_metastore_uris=thrift://127.0.0.1:45151
--hive_metastore_notification_log_poll_period_seconds=1
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--catalog_manager_bg_task_wait_ms=10
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
W20261120 03:01:22.341703  2324 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
W20261120 03:01:22.342336  2324 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20261120 03:01:22.342787  2324 flags.cc:432] Enabled unsafe flag: --never_fsync=true
W20261120 03:01:22.376122  2324 flags.cc:432] Enabled experimental flag: --enable_metadata_cleanup_for_deleted_tables_and_tablets=true
W20261120 03:01:22.376430  2324 flags.cc:432] Enabled experimental flag: --metadata_for_deleted_table_and_tablet_reserved_secs=0
W20261120 03:01:22.376710  2324 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
W20261120 03:01:22.376973  2324 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
W20261120 03:01:22.377328  2324 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
W20261120 03:01:22.377718  2324 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
I20261120 03:01:22.417335  2324 master_runner.cc:387] Master server non-default flags:
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/logs
--logbuflevel=-1
--logtostderr=true
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.96.84:36533
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/data
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/wal
--hive_metastore_uris=thrift://127.0.0.1:45151
--catalog_manager_bg_task_wait_ms=10
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.126:46565
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/data/info.pb
--webserver_interface=127.0.96.126
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false

Master server version:
kudu 1.19.0-SNAPSHOT
revision 36b6772d828145f8fbba3c49746c86838683f9ba
build type FASTDEBUG
built by None at 20 Nov 2026 02:45:53 UTC on bdcb31816ec0
build id 14870
TSAN enabled
I20261120 03:01:22.418761  2324 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20261120 03:01:22.420413  2324 file_cache.cc:504] Constructed file cache file cache with capacity 419430
W20261120 03:01:22.431722  2330 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:01:22.432055  2331 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:01:23.672986  2332 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1239 milliseconds
W20261120 03:01:23.673852  2333 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:01:23.676786  2324 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.246s	user 0.505s	sys 0.725s
W20261120 03:01:23.677047  2324 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.246s	user 0.505s	sys 0.725s
I20261120 03:01:23.677297  2324 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20261120 03:01:23.678432  2324 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20261120 03:01:23.680987  2324 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20261120 03:01:23.682337  2324 hybrid_clock.cc:648] HybridClock initialized: now 1795143683682310 us; error 41 us; skew 500 ppm
I20261120 03:01:23.683246  2324 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20261120 03:01:23.690945  2324 webserver.cc:577] Webserver started at http://127.0.96.126:46381/ using document root <none> and password file <none>
I20261120 03:01:23.691851  2324 fs_manager.cc:362] Metadata directory not provided
I20261120 03:01:23.692065  2324 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20261120 03:01:23.692521  2324 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
I20261120 03:01:23.696751  2324 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/data/instance:
uuid: "6e18f436ef9945d78f543c3471320ee7"
format_stamp: "Formatted at 2026-11-20 03:01:23 on dist-test-slave-jpwh"
I20261120 03:01:23.697844  2324 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/wal/instance:
uuid: "6e18f436ef9945d78f543c3471320ee7"
format_stamp: "Formatted at 2026-11-20 03:01:23 on dist-test-slave-jpwh"
I20261120 03:01:23.705899  2324 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.001s	sys 0.005s
I20261120 03:01:23.711750  2341 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:01:23.712999  2324 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
I20261120 03:01:23.713338  2324 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/data,/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/wal
uuid: "6e18f436ef9945d78f543c3471320ee7"
format_stamp: "Formatted at 2026-11-20 03:01:23 on dist-test-slave-jpwh"
I20261120 03:01:23.713675  2324 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/wal
metadata directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/wal
1 data directories: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20261120 03:01:23.779579  2324 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20261120 03:01:23.781113  2324 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20261120 03:01:23.781558  2324 kserver.cc:163] Server-wide thread pool size limit: 3276
I20261120 03:01:23.861559  2324 rpc_server.cc:307] RPC server started. Bound to: 127.0.96.126:46565
I20261120 03:01:23.861636  2392 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.96.126:46565 every 8 connection(s)
I20261120 03:01:23.864388  2324 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/data/info.pb
I20261120 03:01:23.869726   385 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu as pid 2324
I20261120 03:01:23.869853  2393 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:23.870265   385 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/wal/instance
I20261120 03:01:23.872645   385 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
/tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/wal
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/logs
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.125:36419
--webserver_interface=127.0.96.125
--webserver_port=0
--builtin_ntp_servers=127.0.96.84:36533
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751
--hive_metastore_uris=thrift://127.0.0.1:45151
--hive_metastore_notification_log_poll_period_seconds=1
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--catalog_manager_bg_task_wait_ms=10
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
I20261120 03:01:23.880389  2393 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } has no permanent_uuid. Determining permanent_uuid...
I20261120 03:01:23.912420  2393 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } has no permanent_uuid. Determining permanent_uuid...
W20261120 03:01:23.915040  2342 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.125:36419: connect: Connection refused (error 111)
W20261120 03:01:23.919335  2393 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.125:36419: Network error: Client connection negotiation failed: client connection to 127.0.96.125:36419: connect: Connection refused (error 111)
I20261120 03:01:23.968799  2393 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } attempt: 1
W20261120 03:01:23.973138  2393 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.125:36419: Network error: Client connection negotiation failed: client connection to 127.0.96.125:36419: connect: Connection refused (error 111)
I20261120 03:01:24.041544  2393 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } attempt: 2
W20261120 03:01:24.046046  2393 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.125:36419: Network error: Client connection negotiation failed: client connection to 127.0.96.125:36419: connect: Connection refused (error 111)
I20261120 03:01:24.137485  2393 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } attempt: 3
W20261120 03:01:24.141856  2393 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.125:36419: Network error: Client connection negotiation failed: client connection to 127.0.96.125:36419: connect: Connection refused (error 111)
W20261120 03:01:24.183893  2394 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
W20261120 03:01:24.184518  2394 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20261120 03:01:24.184976  2394 flags.cc:432] Enabled unsafe flag: --never_fsync=true
W20261120 03:01:24.218293  2394 flags.cc:432] Enabled experimental flag: --enable_metadata_cleanup_for_deleted_tables_and_tablets=true
W20261120 03:01:24.218566  2394 flags.cc:432] Enabled experimental flag: --metadata_for_deleted_table_and_tablet_reserved_secs=0
W20261120 03:01:24.218827  2394 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
W20261120 03:01:24.219065  2394 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
W20261120 03:01:24.219303  2394 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
W20261120 03:01:24.219540  2394 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
I20261120 03:01:24.256942  2394 master_runner.cc:387] Master server non-default flags:
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/logs
--logbuflevel=-1
--logtostderr=true
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.96.84:36533
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/data
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/wal
--hive_metastore_uris=thrift://127.0.0.1:45151
--catalog_manager_bg_task_wait_ms=10
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.125:36419
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/data/info.pb
--webserver_interface=127.0.96.125
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false

Master server version:
kudu 1.19.0-SNAPSHOT
revision 36b6772d828145f8fbba3c49746c86838683f9ba
build type FASTDEBUG
built by None at 20 Nov 2026 02:45:53 UTC on bdcb31816ec0
build id 14870
TSAN enabled
I20261120 03:01:24.258378  2394 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20261120 03:01:24.259991  2394 file_cache.cc:504] Constructed file cache file cache with capacity 419430
W20261120 03:01:24.271600  2403 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:01:24.285285  2393 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } attempt: 4
W20261120 03:01:24.289214  2393 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.125:36419: Network error: Client connection negotiation failed: client connection to 127.0.96.125:36419: connect: Connection refused (error 111)
I20261120 03:01:24.588641  2393 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } attempt: 5
W20261120 03:01:24.594321  2393 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.125:36419: Network error: Client connection negotiation failed: client connection to 127.0.96.125:36419: connect: Connection refused (error 111)
I20261120 03:01:25.141857  2393 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } attempt: 6
W20261120 03:01:25.150921  2393 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.125:36419: Network error: Client connection negotiation failed: client connection to 127.0.96.125:36419: connect: Connection refused (error 111)
W20261120 03:01:24.272250  2404 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:01:25.403903  2406 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:01:25.406201  2405 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1130 milliseconds
I20261120 03:01:25.406342  2394 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20261120 03:01:25.407541  2394 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20261120 03:01:25.410436  2394 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20261120 03:01:25.411867  2394 hybrid_clock.cc:648] HybridClock initialized: now 1795143685411805 us; error 68 us; skew 500 ppm
I20261120 03:01:25.412772  2394 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20261120 03:01:25.419083  2394 webserver.cc:577] Webserver started at http://127.0.96.125:37771/ using document root <none> and password file <none>
I20261120 03:01:25.420115  2394 fs_manager.cc:362] Metadata directory not provided
I20261120 03:01:25.420351  2394 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20261120 03:01:25.420800  2394 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
I20261120 03:01:25.425189  2394 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/data/instance:
uuid: "a23f052846654c9daf3fb0d90fbb6f09"
format_stamp: "Formatted at 2026-11-20 03:01:25 on dist-test-slave-jpwh"
I20261120 03:01:25.426435  2394 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/wal/instance:
uuid: "a23f052846654c9daf3fb0d90fbb6f09"
format_stamp: "Formatted at 2026-11-20 03:01:25 on dist-test-slave-jpwh"
I20261120 03:01:25.434406  2394 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.001s
I20261120 03:01:25.440268  2415 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:01:25.441565  2394 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.001s
I20261120 03:01:25.441915  2394 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/data,/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/wal
uuid: "a23f052846654c9daf3fb0d90fbb6f09"
format_stamp: "Formatted at 2026-11-20 03:01:25 on dist-test-slave-jpwh"
I20261120 03:01:25.442332  2394 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/wal
metadata directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/wal
1 data directories: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20261120 03:01:25.493469  2394 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20261120 03:01:25.495152  2394 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20261120 03:01:25.495630  2394 kserver.cc:163] Server-wide thread pool size limit: 3276
I20261120 03:01:25.564646  2394 rpc_server.cc:307] RPC server started. Bound to: 127.0.96.125:36419
I20261120 03:01:25.564738  2466 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.96.125:36419 every 8 connection(s)
I20261120 03:01:25.567613  2394 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/data/info.pb
I20261120 03:01:25.573499  2467 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:25.573853   385 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu as pid 2394
I20261120 03:01:25.574285   385 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/wal/instance
I20261120 03:01:25.576834   385 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
/tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-2/wal
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-2/logs
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.124:33751
--webserver_interface=127.0.96.124
--webserver_port=0
--builtin_ntp_servers=127.0.96.84:36533
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751
--hive_metastore_uris=thrift://127.0.0.1:45151
--hive_metastore_notification_log_poll_period_seconds=1
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--catalog_manager_bg_task_wait_ms=10
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
I20261120 03:01:25.584873  2467 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } has no permanent_uuid. Determining permanent_uuid...
I20261120 03:01:25.607710  2467 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } has no permanent_uuid. Determining permanent_uuid...
I20261120 03:01:25.623231  2467 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } has no permanent_uuid. Determining permanent_uuid...
W20261120 03:01:25.625976  2416 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:01:25.630496  2467 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:33751: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:25.680142  2467 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } attempt: 1
W20261120 03:01:25.684526  2467 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:33751: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:25.753016  2467 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } attempt: 2
W20261120 03:01:25.757345  2467 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:33751: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:25.848830  2467 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } attempt: 3
W20261120 03:01:25.853024  2467 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:33751: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:01:25.904291  2468 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
W20261120 03:01:25.904935  2468 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20261120 03:01:25.905370  2468 flags.cc:432] Enabled unsafe flag: --never_fsync=true
W20261120 03:01:25.939431  2468 flags.cc:432] Enabled experimental flag: --enable_metadata_cleanup_for_deleted_tables_and_tablets=true
W20261120 03:01:25.939682  2468 flags.cc:432] Enabled experimental flag: --metadata_for_deleted_table_and_tablet_reserved_secs=0
W20261120 03:01:25.939894  2468 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
W20261120 03:01:25.940155  2468 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
W20261120 03:01:25.940384  2468 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
W20261120 03:01:25.940604  2468 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
I20261120 03:01:25.979961  2468 master_runner.cc:387] Master server non-default flags:
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-2/logs
--logbuflevel=-1
--logtostderr=true
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.96.84:36533
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-2/data
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-2/wal
--hive_metastore_uris=thrift://127.0.0.1:45151
--catalog_manager_bg_task_wait_ms=10
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.124:33751
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-2/data/info.pb
--webserver_interface=127.0.96.124
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false

Master server version:
kudu 1.19.0-SNAPSHOT
revision 36b6772d828145f8fbba3c49746c86838683f9ba
build type FASTDEBUG
built by None at 20 Nov 2026 02:45:53 UTC on bdcb31816ec0
build id 14870
TSAN enabled
I20261120 03:01:25.981344  2468 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20261120 03:01:25.983007  2468 file_cache.cc:504] Constructed file cache file cache with capacity 419430
W20261120 03:01:25.994681  2478 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:01:25.996500  2467 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } attempt: 4
W20261120 03:01:26.001212  2467 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:33751: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:26.211483  2393 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } attempt: 7
I20261120 03:01:26.223516  2393 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } has no permanent_uuid. Determining permanent_uuid...
W20261120 03:01:26.229554  2393 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:33751: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:26.287967  2393 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } attempt: 1
W20261120 03:01:26.292397  2393 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:33751: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:26.300637  2467 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } attempt: 5
W20261120 03:01:26.305140  2467 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:33751: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:26.373803  2393 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } attempt: 2
W20261120 03:01:26.378041  2393 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:33751: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:26.463615  2393 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } attempt: 3
W20261120 03:01:26.468019  2393 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:33751: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:26.608593  2393 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } attempt: 4
W20261120 03:01:26.613502  2393 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:33751: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:26.852667  2467 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } attempt: 6
W20261120 03:01:26.860689  2467 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:33751: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:26.896970  2393 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } attempt: 5
W20261120 03:01:26.901762  2393 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.96.124:33751: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:01:25.995613  2479 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:01:25.998728  2481 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:01:27.107425  2480 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
I20261120 03:01:27.107440  2468 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20261120 03:01:27.111389  2468 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20261120 03:01:27.114327  2468 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20261120 03:01:27.115724  2468 hybrid_clock.cc:648] HybridClock initialized: now 1795143687115682 us; error 64 us; skew 500 ppm
I20261120 03:01:27.116590  2468 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20261120 03:01:27.122511  2468 webserver.cc:577] Webserver started at http://127.0.96.124:45701/ using document root <none> and password file <none>
I20261120 03:01:27.123442  2468 fs_manager.cc:362] Metadata directory not provided
I20261120 03:01:27.123677  2468 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20261120 03:01:27.124153  2468 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
I20261120 03:01:27.129778  2468 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-2/data/instance:
uuid: "86ed298871604dbe84645c0199254db8"
format_stamp: "Formatted at 2026-11-20 03:01:27 on dist-test-slave-jpwh"
I20261120 03:01:27.130908  2468 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-2/wal/instance:
uuid: "86ed298871604dbe84645c0199254db8"
format_stamp: "Formatted at 2026-11-20 03:01:27 on dist-test-slave-jpwh"
I20261120 03:01:27.137995  2468 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.001s
I20261120 03:01:27.143647  2493 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:01:27.144804  2468 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
I20261120 03:01:27.145138  2468 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-2/data,/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-2/wal
uuid: "86ed298871604dbe84645c0199254db8"
format_stamp: "Formatted at 2026-11-20 03:01:27 on dist-test-slave-jpwh"
I20261120 03:01:27.145469  2468 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-2/wal
metadata directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-2/wal
1 data directories: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20261120 03:01:27.193409  2468 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20261120 03:01:27.195014  2468 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20261120 03:01:27.195498  2468 kserver.cc:163] Server-wide thread pool size limit: 3276
I20261120 03:01:27.262828  2468 rpc_server.cc:307] RPC server started. Bound to: 127.0.96.124:33751
I20261120 03:01:27.262892  2544 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.96.124:33751 every 8 connection(s)
I20261120 03:01:27.265832  2468 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-2/data/info.pb
I20261120 03:01:27.271359  2545 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:27.272406   385 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu as pid 2468
I20261120 03:01:27.272817   385 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-2/wal/instance
I20261120 03:01:27.286990  2545 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } has no permanent_uuid. Determining permanent_uuid...
I20261120 03:01:27.311888  2545 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } has no permanent_uuid. Determining permanent_uuid...
I20261120 03:01:27.321600  2545 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } has no permanent_uuid. Determining permanent_uuid...
I20261120 03:01:27.339809  2545 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: Bootstrap starting.
I20261120 03:01:27.347676  2545 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:27.349561  2545 log.cc:826] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: Log is configured to *not* fsync() on all Append() calls
I20261120 03:01:27.354851  2545 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: No bootstrap required, opened a new log
I20261120 03:01:27.372869  2545 raft_consensus.cc:366] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:27.373509  2545 raft_consensus.cc:392] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:27.373744  2545 raft_consensus.cc:787] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 86ed298871604dbe84645c0199254db8, State: Initialized, Role: FOLLOWER
I20261120 03:01:27.374766  2545 consensus_queue.cc:261] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:27.377687  2553 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:27.378383  2553 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:01:27.379357  2545 sys_catalog.cc:565] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [sys.catalog]: configured and running, proceeding with master startup.
W20261120 03:01:27.418252  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.418645  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.433234  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.435601  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.409447  2468 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: OK, instance_id { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" instance_seqno: 1795143683843452 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
I20261120 03:01:27.440176  2468 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:01:27.450140  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.450650  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.450675  2499 tablet.cc:2554] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20261120 03:01:27.454154  2393 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } attempt: 6
W20261120 03:01:27.465454  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.465830  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.480355  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.480988  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.495081  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.495473  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.507879  2393 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: Bootstrap starting.
W20261120 03:01:27.510141  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.510521  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.514430  2393 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:27.516307  2393 log.cc:826] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: Log is configured to *not* fsync() on all Append() calls
I20261120 03:01:27.520583  2393 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: No bootstrap required, opened a new log
W20261120 03:01:27.525007  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.525326  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.539724  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.540063  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.543020  2393 raft_consensus.cc:366] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:27.543640  2393 raft_consensus.cc:392] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:27.543883  2393 raft_consensus.cc:787] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6e18f436ef9945d78f543c3471320ee7, State: Initialized, Role: FOLLOWER
I20261120 03:01:27.544649  2393 consensus_queue.cc:261] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:27.547221  2572 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:27.547992  2572 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:01:27.548599  2393 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: configured and running, proceeding with master startup.
W20261120 03:01:27.556303  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.556717  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.571278  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.571703  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.587497  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.587940  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.602433  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.603853  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.604324  2587 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
W20261120 03:01:27.618468  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.618965  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.623160  2587 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.623605  2587 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.633733  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.634424  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.638998  2587 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.639412  2587 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.612345  2324 master_runner.cc:187] Error getting master registration for 127.0.96.125:36419: OK, instance_id { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" instance_seqno: 1795143685546130 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
I20261120 03:01:27.641669  2324 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:01:27.649150  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.649494  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.654024  2587 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.654395  2587 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.664283  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.664752  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.669377  2587 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.669725  2587 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.679486  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.679838  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.684258  2587 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.684670  2587 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.696413  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.696800  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.702169  2587 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.702554  2587 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.711347  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.711678  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.717033  2587 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.717365  2587 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.726368  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.726753  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.733979  2587 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.734326  2587 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.741288  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.741622  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.748716  2587 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.749090  2587 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.756055  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.756431  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.762620  2572 raft_consensus.cc:534] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:27.763170  2572 raft_consensus.cc:556] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
W20261120 03:01:27.764377  2587 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.764679  2587 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.765827  2572 leader_election.cc:290] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
I20261120 03:01:27.767275  2520 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6e18f436ef9945d78f543c3471320ee7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "86ed298871604dbe84645c0199254db8" is_pre_election: true
I20261120 03:01:27.768297  2520 raft_consensus.cc:2062] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 6e18f436ef9945d78f543c3471320ee7 in term 0
I20261120 03:01:27.770346  2342 leader_election.cc:304] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8; no voters: 
W20261120 03:01:27.770853  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.771131  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.772647  2572 raft_consensus.cc:3024] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:27.773116  2572 raft_consensus.cc:534] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:27.773445  2572 raft_consensus.cc:3296] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:27.774354  2572 raft_consensus.cc:556] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
W20261120 03:01:27.779289  2587 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.779604  2587 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.785585  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.786054  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.791566  2347 tablet.cc:2554] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20261120 03:01:27.794087  2587 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.794396  2587 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.800806  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.801134  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.800714  2520 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6e18f436ef9945d78f543c3471320ee7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "86ed298871604dbe84645c0199254db8"
I20261120 03:01:27.801625  2520 raft_consensus.cc:3296] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:27.803020  2520 raft_consensus.cc:2062] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 6e18f436ef9945d78f543c3471320ee7 in term 1
I20261120 03:01:27.803692  2572 leader_election.cc:290] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 1 election: Requested vote from peers a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:01:27.810257  2587 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.810590  2587 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.815639  2342 leader_election.cc:304] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8; no voters: 
I20261120 03:01:27.818004  2589 raft_consensus.cc:3024] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 1 FOLLOWER]: Leader election won for term 1
W20261120 03:01:27.821157  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.821610  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.822677  2589 raft_consensus.cc:744] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 1 LEADER]: Becoming Leader. State: Replica: 6e18f436ef9945d78f543c3471320ee7, State: Running, Role: LEADER
W20261120 03:01:27.836418  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
I20261120 03:01:27.835134  2589 consensus_queue.cc:238] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
W20261120 03:01:27.836709  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.851980  2572 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6e18f436ef9945d78f543c3471320ee7. Latest consensus state: current_term: 1 leader_uuid: "6e18f436ef9945d78f543c3471320ee7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:27.852725  2572 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: This master's current role is: LEADER
W20261120 03:01:27.854045  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.854353  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.855945  2587 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.856251  2587 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.861097  2592 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
W20261120 03:01:27.868186  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.868561  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.870438  2589 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20261120 03:01:27.872510  2592 catalog_manager.cc:1535] Initializing Kudu cluster ID...
I20261120 03:01:27.902705  2553 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6e18f436ef9945d78f543c3471320ee7. Latest consensus state: current_term: 1 leader_uuid: "6e18f436ef9945d78f543c3471320ee7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:27.903403  2553 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [sys.catalog]: This master's current role is: FOLLOWER
W20261120 03:01:27.908241  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.910125  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.911877  2591 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6e18f436ef9945d78f543c3471320ee7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:27.912755  2591 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: This master's current role is: LEADER
I20261120 03:01:27.921056  2467 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } attempt: 7
W20261120 03:01:27.931221  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.931546  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.932380  2553 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6e18f436ef9945d78f543c3471320ee7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:27.933084  2553 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [sys.catalog]: This master's current role is: FOLLOWER
W20261120 03:01:27.946225  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.946527  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
W20261120 03:01:27.963003  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.963300  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:27.968228  2592 catalog_manager.cc:1398] Generated new cluster ID: 7f28cd38e32845a18797274c93499aa6
I20261120 03:01:27.968502  2592 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
I20261120 03:01:27.976177  2467 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09: Bootstrap starting.
W20261120 03:01:27.997435  2567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20261120 03:01:27.999982  2567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:28.005662  2467 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:28.008152  2467 log.cc:826] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09: Log is configured to *not* fsync() on all Append() calls
I20261120 03:01:28.022126  2567 catalog_manager.cc:1310] Loaded cluster ID: 7f28cd38e32845a18797274c93499aa6
I20261120 03:01:28.022933  2567 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: loading cluster ID for follower catalog manager: success
I20261120 03:01:28.025367  2467 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09: No bootstrap required, opened a new log
W20261120 03:01:28.036720  2567 catalog_manager.cc:1627] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: acquiring CA information for follower catalog manager: Not found: root CA entry not found
W20261120 03:01:28.037055  2567 catalog_manager.cc:898] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry
I20261120 03:01:28.044782  2592 catalog_manager.cc:1421] Generated new certificate authority record
I20261120 03:01:28.046887  2592 catalog_manager.cc:1555] Loading token signing keys...
I20261120 03:01:28.056789  2467 raft_consensus.cc:366] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:28.057572  2467 raft_consensus.cc:392] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:28.057842  2467 raft_consensus.cc:787] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a23f052846654c9daf3fb0d90fbb6f09, State: Initialized, Role: FOLLOWER
I20261120 03:01:28.058764  2467 consensus_queue.cc:261] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:28.069989  2467 sys_catalog.cc:565] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: configured and running, proceeding with master startup.
I20261120 03:01:28.070168  2441 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6e18f436ef9945d78f543c3471320ee7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a23f052846654c9daf3fb0d90fbb6f09"
I20261120 03:01:28.070916  2441 raft_consensus.cc:3296] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:28.073274  2442 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6e18f436ef9945d78f543c3471320ee7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a23f052846654c9daf3fb0d90fbb6f09" is_pre_election: true
I20261120 03:01:28.067076  2600 sys_catalog.cc:455] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:28.076195  2600 sys_catalog.cc:458] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:01:28.076928  2567 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: acquiring CA information for follower catalog manager: success
W20261120 03:01:28.082511  2567 catalog_manager.cc:898] Not found: no TSK found in the system table: failed to prepare follower catalog manager, will retry
I20261120 03:01:28.091441  2441 raft_consensus.cc:2062] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 6e18f436ef9945d78f543c3471320ee7 in term 1
I20261120 03:01:28.091445  2591 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:28.113569  2592 catalog_manager.cc:6237] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: Generated new TSK 0
I20261120 03:01:28.114816  2592 catalog_manager.cc:1565] Initializing in-progress tserver states...
W20261120 03:01:28.119892  2567 catalog_manager.cc:898] Not found: no TSK found in the system table: failed to prepare follower catalog manager, will retry
I20261120 03:01:28.120208  2592 catalog_manager.cc:1578] Loading latest processed Hive Metastore notification log event ID...
I20261120 03:01:28.126852  2592 catalog_manager.cc:6086] Last processed Hive Metastore notification event ID: -1
I20261120 03:01:28.137205  2567 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
I20261120 03:01:28.152318   385 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
/tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/wal
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/logs
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.96.65:0
--local_ip_for_outbound_sockets=127.0.96.65
--webserver_interface=127.0.96.65
--webserver_port=0
--tserver_master_addrs=127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751
--builtin_ntp_servers=127.0.96.84:36533
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--heartbeat_rpc_timeout_ms=1000
--heartbeat_max_failures_before_backoff=1
--heartbeat_interval_ms=500 with env {}
I20261120 03:01:28.164345  2606 mvcc.cc:204] Tried to move back new op lower bound from 7352908545730805760 to 7352908545396531200. Current Snapshot: MvccSnapshot[applied={T|T < 7352908545730805760}]
I20261120 03:01:28.181370  2600 sys_catalog.cc:455] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6e18f436ef9945d78f543c3471320ee7. Latest consensus state: current_term: 1 leader_uuid: "6e18f436ef9945d78f543c3471320ee7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:28.182087  2600 sys_catalog.cc:458] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:01:28.182129  2589 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6e18f436ef9945d78f543c3471320ee7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:28.182778  2589 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: This master's current role is: LEADER
I20261120 03:01:28.183295  2601 sys_catalog.cc:455] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6e18f436ef9945d78f543c3471320ee7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:28.184012  2601 sys_catalog.cc:458] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:01:28.212738  2621 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
I20261120 03:01:28.240911  2621 catalog_manager.cc:1310] Loaded cluster ID: 7f28cd38e32845a18797274c93499aa6
I20261120 03:01:28.241291  2621 catalog_manager.cc:1603] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09: loading cluster ID for follower catalog manager: success
I20261120 03:01:28.248898  2621 catalog_manager.cc:1625] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09: acquiring CA information for follower catalog manager: success
I20261120 03:01:28.255105  2621 catalog_manager.cc:1653] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
I20261120 03:01:28.638433  2623 raft_consensus.cc:534] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:28.639264  2623 raft_consensus.cc:556] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:28.644925  2368 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86ed298871604dbe84645c0199254db8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6e18f436ef9945d78f543c3471320ee7" is_pre_election: true
I20261120 03:01:28.650182  2623 leader_election.cc:290] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419)
I20261120 03:01:28.651695  2440 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86ed298871604dbe84645c0199254db8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a23f052846654c9daf3fb0d90fbb6f09" is_pre_election: true
I20261120 03:01:28.653864  2494 leader_election.cc:304] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 86ed298871604dbe84645c0199254db8; no voters: 6e18f436ef9945d78f543c3471320ee7, a23f052846654c9daf3fb0d90fbb6f09
I20261120 03:01:28.654850  2623 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20261120 03:01:28.759112  2612 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
W20261120 03:01:28.759793  2612 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20261120 03:01:28.760756  2612 flags.cc:432] Enabled unsafe flag: --never_fsync=true
W20261120 03:01:28.824357  2612 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
W20261120 03:01:28.826056  2612 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.96.65
I20261120 03:01:28.896833  2612 tablet_server_runner.cc:78] Tablet server non-default flags:
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/logs
--logbuflevel=-1
--logtostderr=true
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.96.84:36533
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/data
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.0.96.65:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/data/info.pb
--webserver_interface=127.0.96.65
--webserver_port=0
--heartbeat_interval_ms=500
--heartbeat_max_failures_before_backoff=1
--heartbeat_rpc_timeout_ms=1000
--tserver_master_addrs=127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.0.96.65

Tablet server version:
kudu 1.19.0-SNAPSHOT
revision 36b6772d828145f8fbba3c49746c86838683f9ba
build type FASTDEBUG
built by None at 20 Nov 2026 02:45:53 UTC on bdcb31816ec0
build id 14870
TSAN enabled
I20261120 03:01:28.898654  2612 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20261120 03:01:28.901077  2612 file_cache.cc:504] Constructed file cache file cache with capacity 419430
W20261120 03:01:28.950687  2629 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:01:30.001799  2633 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:30.002506  2633 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:30.005462  2633 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
I20261120 03:01:30.006385  2368 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a23f052846654c9daf3fb0d90fbb6f09" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6e18f436ef9945d78f543c3471320ee7" is_pre_election: true
I20261120 03:01:30.007257  2520 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a23f052846654c9daf3fb0d90fbb6f09" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "86ed298871604dbe84645c0199254db8" is_pre_election: true
I20261120 03:01:30.008878  2416 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:01:30.011387  2633 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20261120 03:01:30.207180  2634 raft_consensus.cc:534] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:30.210199  2634 raft_consensus.cc:556] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:30.214897  2368 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86ed298871604dbe84645c0199254db8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6e18f436ef9945d78f543c3471320ee7" is_pre_election: true
I20261120 03:01:30.219336  2440 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86ed298871604dbe84645c0199254db8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a23f052846654c9daf3fb0d90fbb6f09" is_pre_election: true
I20261120 03:01:30.220981  2494 leader_election.cc:304] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 86ed298871604dbe84645c0199254db8; no voters: 6e18f436ef9945d78f543c3471320ee7, a23f052846654c9daf3fb0d90fbb6f09
I20261120 03:01:30.233873  2634 leader_election.cc:290] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419)
I20261120 03:01:30.236369  2635 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20261120 03:01:30.610188  2636 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:30.611848  2636 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:30.624277  2368 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a23f052846654c9daf3fb0d90fbb6f09" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6e18f436ef9945d78f543c3471320ee7" is_pre_election: true
I20261120 03:01:30.629142  2520 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a23f052846654c9daf3fb0d90fbb6f09" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "86ed298871604dbe84645c0199254db8" is_pre_election: true
I20261120 03:01:30.631327  2416 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:01:30.642701  2636 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
I20261120 03:01:30.643067  2637 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20261120 03:01:31.141605  2638 raft_consensus.cc:534] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:31.146378  2638 raft_consensus.cc:556] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:31.149685  2368 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86ed298871604dbe84645c0199254db8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6e18f436ef9945d78f543c3471320ee7" is_pre_election: true
I20261120 03:01:31.153366  2440 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86ed298871604dbe84645c0199254db8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a23f052846654c9daf3fb0d90fbb6f09" is_pre_election: true
I20261120 03:01:31.155193  2494 leader_election.cc:304] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 86ed298871604dbe84645c0199254db8; no voters: 6e18f436ef9945d78f543c3471320ee7, a23f052846654c9daf3fb0d90fbb6f09
I20261120 03:01:31.160787  2638 leader_election.cc:290] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419)
I20261120 03:01:31.162257  2638 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20261120 03:01:31.235962  2640 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:31.236706  2640 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:31.240496  2640 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
I20261120 03:01:31.240502  2368 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a23f052846654c9daf3fb0d90fbb6f09" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6e18f436ef9945d78f543c3471320ee7" is_pre_election: true
I20261120 03:01:31.241684  2520 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a23f052846654c9daf3fb0d90fbb6f09" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "86ed298871604dbe84645c0199254db8" is_pre_election: true
I20261120 03:01:31.243336  2418 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:01:31.244074  2640 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20261120 03:01:28.951614  2630 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:01:30.336565  2628 debug-util.cc:398] Leaking SignalData structure 0x7b0800032a00 after lost signal to thread 2612
W20261120 03:01:31.360775  2612 thread.cc:642] GCE (cloud detector) Time spent creating pthread: real 2.411s	user 0.868s	sys 1.158s
W20261120 03:01:31.362898  2612 thread.cc:609] GCE (cloud detector) Time spent starting thread: real 2.413s	user 0.868s	sys 1.158s
W20261120 03:01:31.376036  2642 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:01:31.376569  2612 server_base.cc:1061] running on GCE node
I20261120 03:01:31.377914  2612 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20261120 03:01:31.380652  2612 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20261120 03:01:31.382113  2612 hybrid_clock.cc:648] HybridClock initialized: now 1795143691381998 us; error 88 us; skew 500 ppm
I20261120 03:01:31.383165  2612 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20261120 03:01:31.394290  2612 webserver.cc:577] Webserver started at http://127.0.96.65:39553/ using document root <none> and password file <none>
I20261120 03:01:31.395553  2612 fs_manager.cc:362] Metadata directory not provided
I20261120 03:01:31.395829  2612 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20261120 03:01:31.396384  2612 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
I20261120 03:01:31.402742  2612 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/data/instance:
uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd"
format_stamp: "Formatted at 2026-11-20 03:01:31 on dist-test-slave-jpwh"
I20261120 03:01:31.404294  2612 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/wal/instance:
uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd"
format_stamp: "Formatted at 2026-11-20 03:01:31 on dist-test-slave-jpwh"
I20261120 03:01:31.414116  2612 fs_manager.cc:696] Time spent creating directory manager: real 0.009s	user 0.005s	sys 0.005s
I20261120 03:01:31.422739  2647 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:01:31.424211  2612 fs_manager.cc:730] Time spent opening block manager: real 0.006s	user 0.004s	sys 0.000s
I20261120 03:01:31.424607  2612 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/data,/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/wal
uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd"
format_stamp: "Formatted at 2026-11-20 03:01:31 on dist-test-slave-jpwh"
I20261120 03:01:31.425042  2612 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/wal
metadata directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/wal
1 data directories: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20261120 03:01:31.487377  2612 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20261120 03:01:31.489310  2612 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20261120 03:01:31.489843  2612 kserver.cc:163] Server-wide thread pool size limit: 3276
I20261120 03:01:31.493636  2612 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20261120 03:01:31.499473  2612 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20261120 03:01:31.499763  2612 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:01:31.500062  2612 ts_tablet_manager.cc:662] Registered 0 tablets
I20261120 03:01:31.500267  2612 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:01:31.714838  2612 rpc_server.cc:307] RPC server started. Bound to: 127.0.96.65:32769
I20261120 03:01:31.714991  2759 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.96.65:32769 every 8 connection(s)
I20261120 03:01:31.717499  2612 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/data/info.pb
I20261120 03:01:31.719030   385 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu as pid 2612
I20261120 03:01:31.719515   385 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/wal/instance
I20261120 03:01:31.737107   385 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
/tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/wal
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/logs
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.96.66:0
--local_ip_for_outbound_sockets=127.0.96.66
--webserver_interface=127.0.96.66
--webserver_port=0
--tserver_master_addrs=127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751
--builtin_ntp_servers=127.0.96.84:36533
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--heartbeat_rpc_timeout_ms=1000
--heartbeat_max_failures_before_backoff=1
--heartbeat_interval_ms=500 with env {}
I20261120 03:01:31.759438  2765 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:31.760008  2765 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:31.763032  2368 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a23f052846654c9daf3fb0d90fbb6f09" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6e18f436ef9945d78f543c3471320ee7" is_pre_election: true
I20261120 03:01:31.763163  2765 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
I20261120 03:01:31.772202  2520 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a23f052846654c9daf3fb0d90fbb6f09" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "86ed298871604dbe84645c0199254db8" is_pre_election: true
I20261120 03:01:31.776337  2416 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:01:31.778177  2765 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20261120 03:01:31.794240  2760 heartbeater.cc:345] Connected to a master server at 127.0.96.124:33751
I20261120 03:01:31.794790  2760 heartbeater.cc:462] Registering TS with master...
I20261120 03:01:31.796111  2760 heartbeater.cc:508] Master 127.0.96.124:33751 requested a full tablet report, sending...
I20261120 03:01:31.797293  2761 heartbeater.cc:345] Connected to a master server at 127.0.96.125:36419
I20261120 03:01:31.797626  2761 heartbeater.cc:462] Registering TS with master...
I20261120 03:01:31.798489  2761 heartbeater.cc:508] Master 127.0.96.125:36419 requested a full tablet report, sending...
I20261120 03:01:31.799702  2510 ts_manager.cc:210] Registered new tserver with Master: d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769)
I20261120 03:01:31.801997  2432 ts_manager.cc:210] Registered new tserver with Master: d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769)
I20261120 03:01:31.806118  2766 heartbeater.cc:345] Connected to a master server at 127.0.96.126:46565
I20261120 03:01:31.806465  2766 heartbeater.cc:462] Registering TS with master...
I20261120 03:01:31.807330  2766 heartbeater.cc:508] Master 127.0.96.126:46565 requested a full tablet report, sending...
I20261120 03:01:31.809504  2357 ts_manager.cc:210] Registered new tserver with Master: d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769)
I20261120 03:01:31.811355  2357 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.0.96.65:41519
W20261120 03:01:32.096522  2764 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
W20261120 03:01:32.097060  2764 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20261120 03:01:32.097661  2764 flags.cc:432] Enabled unsafe flag: --never_fsync=true
W20261120 03:01:32.131616  2764 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
W20261120 03:01:32.132572  2764 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.96.66
I20261120 03:01:32.173301  2764 tablet_server_runner.cc:78] Tablet server non-default flags:
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/logs
--logbuflevel=-1
--logtostderr=true
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.96.84:36533
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/data
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.0.96.66:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/data/info.pb
--webserver_interface=127.0.96.66
--webserver_port=0
--heartbeat_interval_ms=500
--heartbeat_max_failures_before_backoff=1
--heartbeat_rpc_timeout_ms=1000
--tserver_master_addrs=127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.0.96.66

Tablet server version:
kudu 1.19.0-SNAPSHOT
revision 36b6772d828145f8fbba3c49746c86838683f9ba
build type FASTDEBUG
built by None at 20 Nov 2026 02:45:53 UTC on bdcb31816ec0
build id 14870
TSAN enabled
I20261120 03:01:32.174690  2764 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20261120 03:01:32.176402  2764 file_cache.cc:504] Constructed file cache file cache with capacity 419430
W20261120 03:01:32.189257  2777 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:01:32.314723  2766 heartbeater.cc:500] Master 127.0.96.126:46565 was elected leader, sending a full tablet report...
I20261120 03:01:32.663043  2784 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:32.663707  2784 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:32.666841  2368 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a23f052846654c9daf3fb0d90fbb6f09" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6e18f436ef9945d78f543c3471320ee7" is_pre_election: true
I20261120 03:01:32.667429  2520 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a23f052846654c9daf3fb0d90fbb6f09" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "86ed298871604dbe84645c0199254db8" is_pre_election: true
I20261120 03:01:32.671038  2416 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:01:32.676899  2784 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
I20261120 03:01:32.677506  2784 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20261120 03:01:33.346807  2787 raft_consensus.cc:534] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:33.350368  2787 raft_consensus.cc:556] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:33.361433  2368 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86ed298871604dbe84645c0199254db8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6e18f436ef9945d78f543c3471320ee7" is_pre_election: true
I20261120 03:01:33.365670  2440 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86ed298871604dbe84645c0199254db8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a23f052846654c9daf3fb0d90fbb6f09" is_pre_election: true
I20261120 03:01:33.367550  2494 leader_election.cc:304] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 86ed298871604dbe84645c0199254db8; no voters: 6e18f436ef9945d78f543c3471320ee7, a23f052846654c9daf3fb0d90fbb6f09
I20261120 03:01:33.376309  2787 leader_election.cc:290] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419)
I20261120 03:01:33.378749  2787 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20261120 03:01:32.190716  2778 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:01:33.508958  2780 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:01:33.511329  2764 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.321s	user 0.416s	sys 0.873s
W20261120 03:01:33.511657  2764 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.321s	user 0.416s	sys 0.873s
W20261120 03:01:33.511739  2779 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1317 milliseconds
I20261120 03:01:33.511933  2764 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20261120 03:01:33.513175  2764 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20261120 03:01:33.515442  2764 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20261120 03:01:33.516813  2764 hybrid_clock.cc:648] HybridClock initialized: now 1795143693516772 us; error 42 us; skew 500 ppm
I20261120 03:01:33.517683  2764 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20261120 03:01:33.525275  2764 webserver.cc:577] Webserver started at http://127.0.96.66:32993/ using document root <none> and password file <none>
I20261120 03:01:33.526254  2764 fs_manager.cc:362] Metadata directory not provided
I20261120 03:01:33.526475  2764 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20261120 03:01:33.526999  2764 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
I20261120 03:01:33.531538  2764 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/data/instance:
uuid: "23741ccd042d4ca49afc3b8d1e26e22c"
format_stamp: "Formatted at 2026-11-20 03:01:33 on dist-test-slave-jpwh"
I20261120 03:01:33.532703  2764 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/wal/instance:
uuid: "23741ccd042d4ca49afc3b8d1e26e22c"
format_stamp: "Formatted at 2026-11-20 03:01:33 on dist-test-slave-jpwh"
I20261120 03:01:33.542047  2764 fs_manager.cc:696] Time spent creating directory manager: real 0.009s	user 0.009s	sys 0.001s
I20261120 03:01:33.548852  2793 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:01:33.550331  2764 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.003s
I20261120 03:01:33.550685  2764 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/data,/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/wal
uuid: "23741ccd042d4ca49afc3b8d1e26e22c"
format_stamp: "Formatted at 2026-11-20 03:01:33 on dist-test-slave-jpwh"
I20261120 03:01:33.551021  2764 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/wal
metadata directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/wal
1 data directories: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20261120 03:01:33.599637  2794 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:33.600323  2794 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:33.603572  2368 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a23f052846654c9daf3fb0d90fbb6f09" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6e18f436ef9945d78f543c3471320ee7" is_pre_election: true
I20261120 03:01:33.605123  2520 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a23f052846654c9daf3fb0d90fbb6f09" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "86ed298871604dbe84645c0199254db8" is_pre_election: true
I20261120 03:01:33.607605  2416 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:01:33.610196  2794 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
I20261120 03:01:33.610738  2794 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20261120 03:01:33.643158  2764 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20261120 03:01:33.644848  2764 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20261120 03:01:33.645287  2764 kserver.cc:163] Server-wide thread pool size limit: 3276
I20261120 03:01:33.648202  2764 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20261120 03:01:33.653031  2764 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20261120 03:01:33.653251  2764 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:01:33.653486  2764 ts_tablet_manager.cc:662] Registered 0 tablets
I20261120 03:01:33.653651  2764 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:01:33.795491  2764 rpc_server.cc:307] RPC server started. Bound to: 127.0.96.66:43733
I20261120 03:01:33.795586  2907 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.96.66:43733 every 8 connection(s)
I20261120 03:01:33.798238  2764 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/data/info.pb
I20261120 03:01:33.806349   385 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu as pid 2764
I20261120 03:01:33.807224   385 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/wal/instance
I20261120 03:01:33.819130   385 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
/tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/wal
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/logs
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.96.67:0
--local_ip_for_outbound_sockets=127.0.96.67
--webserver_interface=127.0.96.67
--webserver_port=0
--tserver_master_addrs=127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751
--builtin_ntp_servers=127.0.96.84:36533
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--heartbeat_rpc_timeout_ms=1000
--heartbeat_max_failures_before_backoff=1
--heartbeat_interval_ms=500 with env {}
I20261120 03:01:33.862416  2908 heartbeater.cc:345] Connected to a master server at 127.0.96.124:33751
I20261120 03:01:33.862936  2908 heartbeater.cc:462] Registering TS with master...
I20261120 03:01:33.864337  2908 heartbeater.cc:508] Master 127.0.96.124:33751 requested a full tablet report, sending...
I20261120 03:01:33.865187  2910 heartbeater.cc:345] Connected to a master server at 127.0.96.126:46565
I20261120 03:01:33.865506  2910 heartbeater.cc:462] Registering TS with master...
I20261120 03:01:33.866387  2910 heartbeater.cc:508] Master 127.0.96.126:46565 requested a full tablet report, sending...
I20261120 03:01:33.868724  2509 ts_manager.cc:210] Registered new tserver with Master: 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733)
I20261120 03:01:33.868798  2357 ts_manager.cc:210] Registered new tserver with Master: 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733)
I20261120 03:01:33.870663  2357 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.0.96.66:36323
I20261120 03:01:33.882481  2909 heartbeater.cc:345] Connected to a master server at 127.0.96.125:36419
I20261120 03:01:33.882906  2909 heartbeater.cc:462] Registering TS with master...
I20261120 03:01:33.883579  2909 heartbeater.cc:508] Master 127.0.96.125:36419 requested a full tablet report, sending...
I20261120 03:01:33.886421  2432 ts_manager.cc:210] Registered new tserver with Master: 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733)
I20261120 03:01:33.920969  2919 raft_consensus.cc:534] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:33.921352  2919 raft_consensus.cc:556] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:33.922855  2919 leader_election.cc:290] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419)
I20261120 03:01:33.923844  2368 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86ed298871604dbe84645c0199254db8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6e18f436ef9945d78f543c3471320ee7" is_pre_election: true
I20261120 03:01:33.923939  2440 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86ed298871604dbe84645c0199254db8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a23f052846654c9daf3fb0d90fbb6f09" is_pre_election: true
I20261120 03:01:33.925262  2494 leader_election.cc:304] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 86ed298871604dbe84645c0199254db8; no voters: 6e18f436ef9945d78f543c3471320ee7, a23f052846654c9daf3fb0d90fbb6f09
I20261120 03:01:33.925838  2919 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20261120 03:01:34.185281  2916 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
W20261120 03:01:34.185788  2916 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20261120 03:01:34.186542  2916 flags.cc:432] Enabled unsafe flag: --never_fsync=true
W20261120 03:01:34.221684  2916 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
W20261120 03:01:34.222702  2916 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.96.67
I20261120 03:01:34.261518  2916 tablet_server_runner.cc:78] Tablet server non-default flags:
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/logs
--logbuflevel=-1
--logtostderr=true
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.96.84:36533
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--raft_heartbeat_interval_ms=1000
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/data
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.0.96.67:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/data/info.pb
--webserver_interface=127.0.96.67
--webserver_port=0
--heartbeat_interval_ms=500
--heartbeat_max_failures_before_backoff=1
--heartbeat_rpc_timeout_ms=1000
--tserver_master_addrs=127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.0.96.67

Tablet server version:
kudu 1.19.0-SNAPSHOT
revision 36b6772d828145f8fbba3c49746c86838683f9ba
build type FASTDEBUG
built by None at 20 Nov 2026 02:45:53 UTC on bdcb31816ec0
build id 14870
TSAN enabled
I20261120 03:01:34.262944  2916 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20261120 03:01:34.264605  2916 file_cache.cc:504] Constructed file cache file cache with capacity 419430
W20261120 03:01:34.278923  2926 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:01:34.373871  2910 heartbeater.cc:500] Master 127.0.96.126:46565 was elected leader, sending a full tablet report...
I20261120 03:01:34.553464  2932 raft_consensus.cc:534] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:34.554047  2932 raft_consensus.cc:556] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:34.556325  2932 leader_election.cc:290] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419)
I20261120 03:01:34.557732  2368 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86ed298871604dbe84645c0199254db8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6e18f436ef9945d78f543c3471320ee7" is_pre_election: true
I20261120 03:01:34.558007  2440 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86ed298871604dbe84645c0199254db8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a23f052846654c9daf3fb0d90fbb6f09" is_pre_election: true
I20261120 03:01:34.560674  2494 leader_election.cc:304] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 86ed298871604dbe84645c0199254db8; no voters: 6e18f436ef9945d78f543c3471320ee7, a23f052846654c9daf3fb0d90fbb6f09
I20261120 03:01:34.561707  2932 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20261120 03:01:34.281742  2927 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:01:34.288973  2929 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:01:35.466511  2928 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1183 milliseconds
I20261120 03:01:35.466619  2916 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20261120 03:01:35.467841  2916 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20261120 03:01:35.470671  2916 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20261120 03:01:35.472115  2916 hybrid_clock.cc:648] HybridClock initialized: now 1795143695472048 us; error 74 us; skew 500 ppm
I20261120 03:01:35.472916  2916 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20261120 03:01:35.479050  2916 webserver.cc:577] Webserver started at http://127.0.96.67:35869/ using document root <none> and password file <none>
I20261120 03:01:35.480225  2916 fs_manager.cc:362] Metadata directory not provided
I20261120 03:01:35.480432  2916 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20261120 03:01:35.480979  2916 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
I20261120 03:01:35.485790  2916 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/data/instance:
uuid: "c393ca4a78704704bc9b58916a81fc3f"
format_stamp: "Formatted at 2026-11-20 03:01:35 on dist-test-slave-jpwh"
I20261120 03:01:35.486969  2916 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/wal/instance:
uuid: "c393ca4a78704704bc9b58916a81fc3f"
format_stamp: "Formatted at 2026-11-20 03:01:35 on dist-test-slave-jpwh"
I20261120 03:01:35.494831  2916 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.007s	sys 0.001s
I20261120 03:01:35.500463  2937 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:01:35.502146  2916 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.001s
I20261120 03:01:35.502441  2916 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/data,/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/wal
uuid: "c393ca4a78704704bc9b58916a81fc3f"
format_stamp: "Formatted at 2026-11-20 03:01:35 on dist-test-slave-jpwh"
I20261120 03:01:35.502759  2916 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/wal
metadata directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/wal
1 data directories: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20261120 03:01:35.553426  2916 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20261120 03:01:35.555151  2916 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20261120 03:01:35.555600  2916 kserver.cc:163] Server-wide thread pool size limit: 3276
I20261120 03:01:35.558470  2916 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
I20261120 03:01:35.562847  2916 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20261120 03:01:35.563069  2916 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:01:35.563318  2916 ts_tablet_manager.cc:662] Registered 0 tablets
I20261120 03:01:35.563474  2916 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:01:35.698971  2916 rpc_server.cc:307] RPC server started. Bound to: 127.0.96.67:40993
I20261120 03:01:35.699071  3049 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.96.67:40993 every 8 connection(s)
I20261120 03:01:35.701778  2916 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/data/info.pb
I20261120 03:01:35.712093   385 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu as pid 2916
I20261120 03:01:35.712612   385 external_mini_cluster.cc:1448] Reading /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/wal/instance
I20261120 03:01:35.748703  3050 heartbeater.cc:345] Connected to a master server at 127.0.96.124:33751
I20261120 03:01:35.749212  3050 heartbeater.cc:462] Registering TS with master...
I20261120 03:01:35.749269  3052 heartbeater.cc:345] Connected to a master server at 127.0.96.126:46565
I20261120 03:01:35.749634  3052 heartbeater.cc:462] Registering TS with master...
I20261120 03:01:35.750675  3050 heartbeater.cc:508] Master 127.0.96.124:33751 requested a full tablet report, sending...
I20261120 03:01:35.750801  3051 heartbeater.cc:345] Connected to a master server at 127.0.96.125:36419
I20261120 03:01:35.750977  3052 heartbeater.cc:508] Master 127.0.96.126:46565 requested a full tablet report, sending...
I20261120 03:01:35.751147  3051 heartbeater.cc:462] Registering TS with master...
I20261120 03:01:35.752045  3051 heartbeater.cc:508] Master 127.0.96.125:36419 requested a full tablet report, sending...
I20261120 03:01:35.753504  2357 ts_manager.cc:210] Registered new tserver with Master: c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993)
I20261120 03:01:35.753825  2509 ts_manager.cc:210] Registered new tserver with Master: c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993)
I20261120 03:01:35.754827  2432 ts_manager.cc:210] Registered new tserver with Master: c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993)
I20261120 03:01:35.755334  2357 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.0.96.67:44741
I20261120 03:01:35.767978   385 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
I20261120 03:01:35.815418  2357 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35970:
name: "default.Table_a938daa15a8b429190ccb7a61dab9f86"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:35.816588  2358 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35970:
name: "default.Table_1c9622c0bec4498cb886d15294dc5395"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:35.819824  2356 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35970:
name: "default.Table_ad2a1b8b11fe4111b57b8eeae744cf96"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:35.821539  2355 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35970:
name: "default.Table_e387307c289c48c392b4a49829f76842"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:01:35.828948  2357 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.table_a938daa15a8b429190ccb7a61dab9f86 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:01:35.837078  2356 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.table_ad2a1b8b11fe4111b57b8eeae744cf96 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:01:35.839222  2355 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.table_e387307c289c48c392b4a49829f76842 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:01:35.842571  2358 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.table_1c9622c0bec4498cb886d15294dc5395 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:01:35.855147   385 external_mini_cluster.cc:1664] Killing /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu with pid 2324
I20261120 03:01:35.871068  3084 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:35.871624  3084 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:35.875303  3084 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
I20261120 03:01:35.877823  2520 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a23f052846654c9daf3fb0d90fbb6f09" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "86ed298871604dbe84645c0199254db8" is_pre_election: true
W20261120 03:01:35.891196  2798 connection.cc:573] client connection to 127.0.96.126:46565 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:35.892215  2910 heartbeater.cc:648] Failed to heartbeat to 127.0.96.126:46565 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:35.892987  3062 master_proxy_rpc.cc:203] Re-attempting CreateTable request to leader Master (127.0.96.126:46565)
W20261120 03:01:35.893031  2650 connection.cc:573] client connection to 127.0.96.126:46565 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:35.895682  2418 connection.cc:573] client connection to 127.0.96.126:46565 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:35.896092  2418 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Connection reset by peer (error 104) [suppressed 6 similar messages]
W20261120 03:01:35.897111  2766 heartbeater.cc:648] Failed to heartbeat to 127.0.96.126:46565 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:35.911517  2418 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
I20261120 03:01:35.917562  2418 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:01:35.918551  3084 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20261120 03:01:35.934125  3088 raft_consensus.cc:534] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:35.935056  3088 raft_consensus.cc:556] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:35.941479  2440 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86ed298871604dbe84645c0199254db8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a23f052846654c9daf3fb0d90fbb6f09" is_pre_election: true
I20261120 03:01:35.942241  2440 raft_consensus.cc:2062] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 86ed298871604dbe84645c0199254db8 in term 1
I20261120 03:01:35.942711  3088 leader_election.cc:290] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419)
I20261120 03:01:35.943749  2494 leader_election.cc:304] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 86ed298871604dbe84645c0199254db8, a23f052846654c9daf3fb0d90fbb6f09; no voters: 
I20261120 03:01:35.944610  3088 raft_consensus.cc:3024] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Leader pre-election won for term 2
I20261120 03:01:35.945045  3088 raft_consensus.cc:534] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:35.945691  3088 raft_consensus.cc:3296] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 1 FOLLOWER]: Advancing to term 2
I20261120 03:01:35.952677  3088 raft_consensus.cc:556] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
W20261120 03:01:35.963572  2496 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:01:35.972452  2496 leader_election.cc:336] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
I20261120 03:01:35.976660  3088 leader_election.cc:290] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 election: Requested vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419)
I20261120 03:01:35.981791  2440 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86ed298871604dbe84645c0199254db8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a23f052846654c9daf3fb0d90fbb6f09"
I20261120 03:01:35.982365  2440 raft_consensus.cc:3296] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 1 FOLLOWER]: Advancing to term 2
I20261120 03:01:35.983372  2440 raft_consensus.cc:2062] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 2 FOLLOWER]: Leader election vote request: granting yes vote for candidate 86ed298871604dbe84645c0199254db8 in term 2
W20261120 03:01:35.985680  2496 leader_election.cc:336] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
I20261120 03:01:35.992362  2494 leader_election.cc:304] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 86ed298871604dbe84645c0199254db8, a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7
I20261120 03:01:35.993281  3090 raft_consensus.cc:3024] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 2 FOLLOWER]: Leader election won for term 2
I20261120 03:01:35.993767  3090 raft_consensus.cc:744] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [term 2 LEADER]: Becoming Leader. State: Replica: 86ed298871604dbe84645c0199254db8, State: Running, Role: LEADER
I20261120 03:01:35.994777  3090 consensus_queue.cc:238] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:36.004853  3088 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 86ed298871604dbe84645c0199254db8. Latest consensus state: current_term: 2 leader_uuid: "86ed298871604dbe84645c0199254db8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:36.005718  3088 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [sys.catalog]: This master's current role is: LEADER
I20261120 03:01:36.015218  3093 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
I20261120 03:01:36.017117  2440 raft_consensus.cc:1338] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 2 FOLLOWER]: Refusing update from remote peer 86ed298871604dbe84645c0199254db8: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
I20261120 03:01:36.031327  3088 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20261120 03:01:36.039608  2496 consensus_peers.cc:597] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 -> Peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Couldn't send request to peer 6e18f436ef9945d78f543c3471320ee7. Status: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20261120 03:01:36.067541  3084 sys_catalog.cc:455] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 86ed298871604dbe84645c0199254db8. Latest consensus state: current_term: 2 leader_uuid: "86ed298871604dbe84645c0199254db8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:36.068403  3084 sys_catalog.cc:458] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:01:36.069237  3093 catalog_manager.cc:1535] Initializing Kudu cluster ID...
I20261120 03:01:36.073734  3093 catalog_manager.cc:1310] Loaded cluster ID: 7f28cd38e32845a18797274c93499aa6
I20261120 03:01:36.074050  3093 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
I20261120 03:01:36.079725  3093 catalog_manager.cc:1555] Loading token signing keys...
I20261120 03:01:36.080406  3088 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "86ed298871604dbe84645c0199254db8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:36.081055  3088 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 [sys.catalog]: This master's current role is: LEADER
I20261120 03:01:36.101279  3084 sys_catalog.cc:455] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "86ed298871604dbe84645c0199254db8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:36.101979  3084 sys_catalog.cc:458] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:01:36.104934  3093 catalog_manager.cc:6248] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: Loaded TSK: 0
I20261120 03:01:36.106491  3093 catalog_manager.cc:1565] Initializing in-progress tserver states...
I20261120 03:01:36.120524  3093 catalog_manager.cc:1578] Loading latest processed Hive Metastore notification log event ID...
I20261120 03:01:36.125658  3093 catalog_manager.cc:6086] Last processed Hive Metastore notification event ID: -1
I20261120 03:01:36.149596  2509 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:57466:
name: "default.Table_e387307c289c48c392b4a49829f76842"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:36.151894  2510 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:57466:
name: "default.Table_ad2a1b8b11fe4111b57b8eeae744cf96"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:36.154734  2507 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:57466:
name: "default.Table_a938daa15a8b429190ccb7a61dab9f86"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:36.153714  2508 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:57466:
name: "default.Table_1c9622c0bec4498cb886d15294dc5395"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
W20261120 03:01:36.165508  2510 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.table_ad2a1b8b11fe4111b57b8eeae744cf96 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:01:36.169860  2507 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.table_a938daa15a8b429190ccb7a61dab9f86 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:01:36.172298  2509 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.table_e387307c289c48c392b4a49829f76842 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
W20261120 03:01:36.174429  2508 catalog_manager.cc:7244] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table default.table_1c9622c0bec4498cb886d15294dc5395 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20261120 03:01:36.258719  3052 heartbeater.cc:500] Master 127.0.96.126:46565 was elected leader, sending a full tablet report...
W20261120 03:01:36.281819  3052 heartbeater.cc:648] Failed to heartbeat to 127.0.96.126:46565 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
I20261120 03:01:36.492549  2508 catalog_manager.cc:2218] created HMS catalog entry for table  [id=982cb6e783c64717b3882765a530c3eb]
I20261120 03:01:36.528725  2508 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:57466:
table { table_name: "default.Table_1c9622c0bec4498cb886d15294dc5395" } new_table_name: "default.Table_b9ef32f90b37407999140f1090dc8049" modify_external_catalogs: true
I20261120 03:01:36.614283  2843 tablet_service.cc:1495] Processing CreateTablet for tablet 45460f5605c844d0a28e598f42fdc4cb (DEFAULT_TABLE table=default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:36.617123  2843 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 45460f5605c844d0a28e598f42fdc4cb. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:36.618288  2695 tablet_service.cc:1495] Processing CreateTablet for tablet 45460f5605c844d0a28e598f42fdc4cb (DEFAULT_TABLE table=default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:36.620260  2695 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 45460f5605c844d0a28e598f42fdc4cb. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:36.639508  2985 tablet_service.cc:1495] Processing CreateTablet for tablet 45460f5605c844d0a28e598f42fdc4cb (DEFAULT_TABLE table=default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:36.641433  2985 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 45460f5605c844d0a28e598f42fdc4cb. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:36.643028  2509 catalog_manager.cc:2218] created HMS catalog entry for table  [id=2aefba03f3944dfc9f108a6431187ac3]
I20261120 03:01:36.664475  2509 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:57466:
table { table_name: "default.Table_e387307c289c48c392b4a49829f76842" } new_table_name: "default.Table_11fd9a471c414923bd22c7180c831f74" modify_external_catalogs: true
I20261120 03:01:36.668989  3112 tablet_bootstrap.cc:492] T 45460f5605c844d0a28e598f42fdc4cb P 23741ccd042d4ca49afc3b8d1e26e22c: Bootstrap starting.
I20261120 03:01:36.689203  2506 master_service.cc:965] ReplaceTablet: received request to replace tablet 45460f5605c844d0a28e598f42fdc4cb from {username='slave'} at 127.0.0.1:57466
I20261120 03:01:36.694159  3113 tablet_bootstrap.cc:492] T 45460f5605c844d0a28e598f42fdc4cb P d9ed1d903ef340ac8f1025c2b7df9bdd: Bootstrap starting.
I20261120 03:01:36.694684  2505 master_service.cc:965] ReplaceTablet: received request to replace tablet 45460f5605c844d0a28e598f42fdc4cb from {username='slave'} at 127.0.0.1:57466
I20261120 03:01:36.700533  3114 tablet_bootstrap.cc:492] T 45460f5605c844d0a28e598f42fdc4cb P c393ca4a78704704bc9b58916a81fc3f: Bootstrap starting.
I20261120 03:01:36.701443  3112 tablet_bootstrap.cc:654] T 45460f5605c844d0a28e598f42fdc4cb P 23741ccd042d4ca49afc3b8d1e26e22c: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:36.703807  3112 log.cc:826] T 45460f5605c844d0a28e598f42fdc4cb P 23741ccd042d4ca49afc3b8d1e26e22c: Log is configured to *not* fsync() on all Append() calls
I20261120 03:01:36.710047  2695 tablet_service.cc:1495] Processing CreateTablet for tablet 7c1a821b612b4325bc34a3a16c284b2d (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:36.710826  2843 tablet_service.cc:1495] Processing CreateTablet for tablet 7c1a821b612b4325bc34a3a16c284b2d (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:36.711208  2695 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7c1a821b612b4325bc34a3a16c284b2d. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:36.711902  2843 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7c1a821b612b4325bc34a3a16c284b2d. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:36.723438  2985 tablet_service.cc:1495] Processing CreateTablet for tablet 7c1a821b612b4325bc34a3a16c284b2d (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:36.724581  2985 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7c1a821b612b4325bc34a3a16c284b2d. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:36.726300  3114 tablet_bootstrap.cc:654] T 45460f5605c844d0a28e598f42fdc4cb P c393ca4a78704704bc9b58916a81fc3f: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:36.726325  2504 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:57466:
name: "default.Table_8feab37db9ac453aaa1afb4721505818"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:36.729054  3114 log.cc:826] T 45460f5605c844d0a28e598f42fdc4cb P c393ca4a78704704bc9b58916a81fc3f: Log is configured to *not* fsync() on all Append() calls
I20261120 03:01:36.731957  3113 tablet_bootstrap.cc:654] T 45460f5605c844d0a28e598f42fdc4cb P d9ed1d903ef340ac8f1025c2b7df9bdd: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:36.740437  3113 log.cc:826] T 45460f5605c844d0a28e598f42fdc4cb P d9ed1d903ef340ac8f1025c2b7df9bdd: Log is configured to *not* fsync() on all Append() calls
I20261120 03:01:36.745932  3112 tablet_bootstrap.cc:492] T 45460f5605c844d0a28e598f42fdc4cb P 23741ccd042d4ca49afc3b8d1e26e22c: No bootstrap required, opened a new log
I20261120 03:01:36.746510  3112 ts_tablet_manager.cc:1449] T 45460f5605c844d0a28e598f42fdc4cb P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent bootstrapping tablet: real 0.080s	user 0.009s	sys 0.010s
I20261120 03:01:36.754545  2506 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: Sending DeleteTablet for 3 replicas of tablet 45460f5605c844d0a28e598f42fdc4cb
I20261120 03:01:36.757440  2695 tablet_service.cc:1542] Processing DeleteTablet for tablet 45460f5605c844d0a28e598f42fdc4cb with delete_type TABLET_DATA_DELETED (replaced by tablet f8087040bf3349e1898a858546ff1f04) from {username='slave'} at 127.0.0.1:38150
I20261120 03:01:36.758543  2984 tablet_service.cc:1542] Processing DeleteTablet for tablet 45460f5605c844d0a28e598f42fdc4cb with delete_type TABLET_DATA_DELETED (replaced by tablet f8087040bf3349e1898a858546ff1f04) from {username='slave'} at 127.0.0.1:41990
I20261120 03:01:36.760579  3050 heartbeater.cc:500] Master 127.0.96.124:33751 was elected leader, sending a full tablet report...
I20261120 03:01:36.761492  2506 catalog_manager.cc:6952] ReplaceTablet: tablet 45460f5605c844d0a28e598f42fdc4cb deleted and replaced by tablet f8087040bf3349e1898a858546ff1f04
I20261120 03:01:36.767835  2842 tablet_service.cc:1542] Processing DeleteTablet for tablet 45460f5605c844d0a28e598f42fdc4cb with delete_type TABLET_DATA_DELETED (replaced by tablet f8087040bf3349e1898a858546ff1f04) from {username='slave'} at 127.0.0.1:35818
W20261120 03:01:36.799711  2497 catalog_manager.cc:5061] TS c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993): delete failed for tablet 45460f5605c844d0a28e598f42fdc4cb because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 45460f5605c844d0a28e598f42fdc4cb already in progress: creating tablet
W20261120 03:01:36.804958  2497 catalog_manager.cc:5061] TS d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769): delete failed for tablet 45460f5605c844d0a28e598f42fdc4cb because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 45460f5605c844d0a28e598f42fdc4cb already in progress: creating tablet
I20261120 03:01:36.806874  2695 tablet_service.cc:1495] Processing CreateTablet for tablet f8087040bf3349e1898a858546ff1f04 (DEFAULT_TABLE table=default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:36.808015  2695 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f8087040bf3349e1898a858546ff1f04. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:36.809132  2984 tablet_service.cc:1495] Processing CreateTablet for tablet f8087040bf3349e1898a858546ff1f04 (DEFAULT_TABLE table=default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:36.810366  2984 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f8087040bf3349e1898a858546ff1f04. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:36.811244  2841 tablet_service.cc:1495] Processing CreateTablet for tablet f8087040bf3349e1898a858546ff1f04 (DEFAULT_TABLE table=default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:36.812466  2841 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet f8087040bf3349e1898a858546ff1f04. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:36.826880   385 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
/tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/wal
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/logs
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.126:46565
--webserver_interface=127.0.96.126
--webserver_port=46381
--builtin_ntp_servers=127.0.96.84:36533
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751
--hive_metastore_uris=thrift://127.0.0.1:45151
--hive_metastore_notification_log_poll_period_seconds=1
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--catalog_manager_bg_task_wait_ms=10
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
W20261120 03:01:36.830554  2495 catalog_manager.cc:5061] TS 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): delete failed for tablet 45460f5605c844d0a28e598f42fdc4cb because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 45460f5605c844d0a28e598f42fdc4cb already in progress: creating tablet
I20261120 03:01:36.839800  3113 tablet_bootstrap.cc:492] T 45460f5605c844d0a28e598f42fdc4cb P d9ed1d903ef340ac8f1025c2b7df9bdd: No bootstrap required, opened a new log
I20261120 03:01:36.840375  3113 ts_tablet_manager.cc:1449] T 45460f5605c844d0a28e598f42fdc4cb P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent bootstrapping tablet: real 0.147s	user 0.012s	sys 0.019s
I20261120 03:01:36.844806  3114 tablet_bootstrap.cc:492] T 45460f5605c844d0a28e598f42fdc4cb P c393ca4a78704704bc9b58916a81fc3f: No bootstrap required, opened a new log
I20261120 03:01:36.845346  3114 ts_tablet_manager.cc:1449] T 45460f5605c844d0a28e598f42fdc4cb P c393ca4a78704704bc9b58916a81fc3f: Time spent bootstrapping tablet: real 0.145s	user 0.021s	sys 0.007s
I20261120 03:01:36.841805  3112 raft_consensus.cc:366] T 45460f5605c844d0a28e598f42fdc4cb P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } }
I20261120 03:01:36.858770  3112 raft_consensus.cc:392] T 45460f5605c844d0a28e598f42fdc4cb P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:36.859104  3112 raft_consensus.cc:787] T 45460f5605c844d0a28e598f42fdc4cb P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23741ccd042d4ca49afc3b8d1e26e22c, State: Initialized, Role: FOLLOWER
I20261120 03:01:36.866463  2506 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:57466:
name: "default.Table_35bb29085419456eb14b37a3b5c7e92a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:36.866562  3112 consensus_queue.cc:261] T 45460f5605c844d0a28e598f42fdc4cb P 23741ccd042d4ca49afc3b8d1e26e22c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } }
I20261120 03:01:36.883428  2760 heartbeater.cc:500] Master 127.0.96.124:33751 was elected leader, sending a full tablet report...
I20261120 03:01:36.892400  2908 heartbeater.cc:500] Master 127.0.96.124:33751 was elected leader, sending a full tablet report...
I20261120 03:01:36.892457  2694 tablet_service.cc:1542] Processing DeleteTablet for tablet 45460f5605c844d0a28e598f42fdc4cb with delete_type TABLET_DATA_DELETED (replaced by tablet f8087040bf3349e1898a858546ff1f04) from {username='slave'} at 127.0.0.1:38150
W20261120 03:01:36.894352  2497 catalog_manager.cc:5061] TS d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769): delete failed for tablet 45460f5605c844d0a28e598f42fdc4cb because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 45460f5605c844d0a28e598f42fdc4cb already in progress: creating tablet
I20261120 03:01:36.901368  3112 ts_tablet_manager.cc:1480] T 45460f5605c844d0a28e598f42fdc4cb P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent starting tablet: real 0.155s	user 0.038s	sys 0.020s
I20261120 03:01:36.905010  2841 tablet_service.cc:1542] Processing DeleteTablet for tablet 45460f5605c844d0a28e598f42fdc4cb with delete_type TABLET_DATA_DELETED (replaced by tablet f8087040bf3349e1898a858546ff1f04) from {username='slave'} at 127.0.0.1:35818
I20261120 03:01:36.906039  3120 tablet_replica.cc:333] T 45460f5605c844d0a28e598f42fdc4cb P 23741ccd042d4ca49afc3b8d1e26e22c: stopping tablet replica
I20261120 03:01:36.906901  3120 raft_consensus.cc:2440] T 45460f5605c844d0a28e598f42fdc4cb P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:36.907462  3120 raft_consensus.cc:2469] T 45460f5605c844d0a28e598f42fdc4cb P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:36.909657  3112 tablet_bootstrap.cc:492] T 7c1a821b612b4325bc34a3a16c284b2d P 23741ccd042d4ca49afc3b8d1e26e22c: Bootstrap starting.
I20261120 03:01:36.918768  3120 ts_tablet_manager.cc:1962] T 45460f5605c844d0a28e598f42fdc4cb P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:36.921729  3112 tablet_bootstrap.cc:654] T 7c1a821b612b4325bc34a3a16c284b2d P 23741ccd042d4ca49afc3b8d1e26e22c: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:36.931245  3112 tablet_bootstrap.cc:492] T 7c1a821b612b4325bc34a3a16c284b2d P 23741ccd042d4ca49afc3b8d1e26e22c: No bootstrap required, opened a new log
I20261120 03:01:36.931695  3112 ts_tablet_manager.cc:1449] T 7c1a821b612b4325bc34a3a16c284b2d P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent bootstrapping tablet: real 0.022s	user 0.012s	sys 0.000s
I20261120 03:01:36.934453  3112 raft_consensus.cc:366] T 7c1a821b612b4325bc34a3a16c284b2d P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:36.935137  3112 raft_consensus.cc:392] T 7c1a821b612b4325bc34a3a16c284b2d P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:36.935403  3112 raft_consensus.cc:787] T 7c1a821b612b4325bc34a3a16c284b2d P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23741ccd042d4ca49afc3b8d1e26e22c, State: Initialized, Role: FOLLOWER
I20261120 03:01:36.936336  3112 consensus_queue.cc:261] T 7c1a821b612b4325bc34a3a16c284b2d P 23741ccd042d4ca49afc3b8d1e26e22c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:36.938324  3112 ts_tablet_manager.cc:1480] T 7c1a821b612b4325bc34a3a16c284b2d P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent starting tablet: real 0.006s	user 0.008s	sys 0.000s
I20261120 03:01:36.939219  3112 tablet_bootstrap.cc:492] T f8087040bf3349e1898a858546ff1f04 P 23741ccd042d4ca49afc3b8d1e26e22c: Bootstrap starting.
I20261120 03:01:36.945381  3112 tablet_bootstrap.cc:654] T f8087040bf3349e1898a858546ff1f04 P 23741ccd042d4ca49afc3b8d1e26e22c: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:36.951895  3114 raft_consensus.cc:366] T 45460f5605c844d0a28e598f42fdc4cb P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } }
I20261120 03:01:36.952873  3114 raft_consensus.cc:392] T 45460f5605c844d0a28e598f42fdc4cb P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:36.953181  3114 raft_consensus.cc:787] T 45460f5605c844d0a28e598f42fdc4cb P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c393ca4a78704704bc9b58916a81fc3f, State: Initialized, Role: FOLLOWER
I20261120 03:01:36.961791  2503 master_service.cc:965] ReplaceTablet: received request to replace tablet 7c1a821b612b4325bc34a3a16c284b2d from {username='slave'} at 127.0.0.1:57466
I20261120 03:01:36.966388  3112 tablet_bootstrap.cc:492] T f8087040bf3349e1898a858546ff1f04 P 23741ccd042d4ca49afc3b8d1e26e22c: No bootstrap required, opened a new log
I20261120 03:01:36.967010  3112 ts_tablet_manager.cc:1449] T f8087040bf3349e1898a858546ff1f04 P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent bootstrapping tablet: real 0.028s	user 0.010s	sys 0.007s
I20261120 03:01:36.969785  3112 raft_consensus.cc:366] T f8087040bf3349e1898a858546ff1f04 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:36.970590  3112 raft_consensus.cc:392] T f8087040bf3349e1898a858546ff1f04 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:36.970929  3112 raft_consensus.cc:787] T f8087040bf3349e1898a858546ff1f04 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23741ccd042d4ca49afc3b8d1e26e22c, State: Initialized, Role: FOLLOWER
I20261120 03:01:36.971850  3112 consensus_queue.cc:261] T f8087040bf3349e1898a858546ff1f04 P 23741ccd042d4ca49afc3b8d1e26e22c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:36.975329  3112 ts_tablet_manager.cc:1480] T f8087040bf3349e1898a858546ff1f04 P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent starting tablet: real 0.008s	user 0.006s	sys 0.000s
I20261120 03:01:36.981995  2503 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: Sending DeleteTablet for 3 replicas of tablet 7c1a821b612b4325bc34a3a16c284b2d
I20261120 03:01:36.983690  2503 catalog_manager.cc:6952] ReplaceTablet: tablet 7c1a821b612b4325bc34a3a16c284b2d deleted and replaced by tablet 4858d6a8264141cf805794882d4f0d56
I20261120 03:01:36.984908  3113 raft_consensus.cc:366] T 45460f5605c844d0a28e598f42fdc4cb P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } }
I20261120 03:01:36.985894  3113 raft_consensus.cc:392] T 45460f5605c844d0a28e598f42fdc4cb P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:36.986205  3113 raft_consensus.cc:787] T 45460f5605c844d0a28e598f42fdc4cb P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9ed1d903ef340ac8f1025c2b7df9bdd, State: Initialized, Role: FOLLOWER
I20261120 03:01:36.986487  2695 tablet_service.cc:1542] Processing DeleteTablet for tablet 7c1a821b612b4325bc34a3a16c284b2d with delete_type TABLET_DATA_DELETED (replaced by tablet 4858d6a8264141cf805794882d4f0d56) from {username='slave'} at 127.0.0.1:38150
I20261120 03:01:36.987156  3113 consensus_queue.cc:261] T 45460f5605c844d0a28e598f42fdc4cb P d9ed1d903ef340ac8f1025c2b7df9bdd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } }
I20261120 03:01:36.989073  2985 tablet_service.cc:1542] Processing DeleteTablet for tablet 7c1a821b612b4325bc34a3a16c284b2d with delete_type TABLET_DATA_DELETED (replaced by tablet 4858d6a8264141cf805794882d4f0d56) from {username='slave'} at 127.0.0.1:41990
I20261120 03:01:36.992473  3114 consensus_queue.cc:261] T 45460f5605c844d0a28e598f42fdc4cb P c393ca4a78704704bc9b58916a81fc3f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } }
W20261120 03:01:36.994365  2497 catalog_manager.cc:5061] TS c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993): delete failed for tablet 7c1a821b612b4325bc34a3a16c284b2d because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 7c1a821b612b4325bc34a3a16c284b2d already in progress: creating tablet
W20261120 03:01:36.995112  2497 catalog_manager.cc:5061] TS d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769): delete failed for tablet 7c1a821b612b4325bc34a3a16c284b2d because tablet deleting was already in progress. No further retry: Already present: State transition of tablet 7c1a821b612b4325bc34a3a16c284b2d already in progress: creating tablet
I20261120 03:01:36.996956  2841 tablet_service.cc:1542] Processing DeleteTablet for tablet 7c1a821b612b4325bc34a3a16c284b2d with delete_type TABLET_DATA_DELETED (replaced by tablet 4858d6a8264141cf805794882d4f0d56) from {username='slave'} at 127.0.0.1:35818
I20261120 03:01:37.004293  3114 ts_tablet_manager.cc:1480] T 45460f5605c844d0a28e598f42fdc4cb P c393ca4a78704704bc9b58916a81fc3f: Time spent starting tablet: real 0.159s	user 0.036s	sys 0.000s
I20261120 03:01:37.016215  3114 tablet_bootstrap.cc:492] T f8087040bf3349e1898a858546ff1f04 P c393ca4a78704704bc9b58916a81fc3f: Bootstrap starting.
I20261120 03:01:37.020136  3113 ts_tablet_manager.cc:1480] T 45460f5605c844d0a28e598f42fdc4cb P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent starting tablet: real 0.179s	user 0.037s	sys 0.001s
I20261120 03:01:37.026823  3113 tablet_bootstrap.cc:492] T 7c1a821b612b4325bc34a3a16c284b2d P d9ed1d903ef340ac8f1025c2b7df9bdd: Bootstrap starting.
I20261120 03:01:37.028065  2985 tablet_service.cc:1542] Processing DeleteTablet for tablet 45460f5605c844d0a28e598f42fdc4cb with delete_type TABLET_DATA_DELETED (replaced by tablet f8087040bf3349e1898a858546ff1f04) from {username='slave'} at 127.0.0.1:41990
I20261120 03:01:37.028975  3119 tablet_replica.cc:333] T 45460f5605c844d0a28e598f42fdc4cb P c393ca4a78704704bc9b58916a81fc3f: stopping tablet replica
I20261120 03:01:37.029807  3119 raft_consensus.cc:2440] T 45460f5605c844d0a28e598f42fdc4cb P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:37.030405  3119 raft_consensus.cc:2469] T 45460f5605c844d0a28e598f42fdc4cb P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:37.033030  3113 tablet_bootstrap.cc:654] T 7c1a821b612b4325bc34a3a16c284b2d P d9ed1d903ef340ac8f1025c2b7df9bdd: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:37.036615  3119 ts_tablet_manager.cc:1962] T 45460f5605c844d0a28e598f42fdc4cb P c393ca4a78704704bc9b58916a81fc3f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:37.037590  2695 tablet_service.cc:1542] Processing DeleteTablet for tablet 45460f5605c844d0a28e598f42fdc4cb with delete_type TABLET_DATA_DELETED (replaced by tablet f8087040bf3349e1898a858546ff1f04) from {username='slave'} at 127.0.0.1:38150
I20261120 03:01:37.038566  3118 tablet_replica.cc:333] T 45460f5605c844d0a28e598f42fdc4cb P d9ed1d903ef340ac8f1025c2b7df9bdd: stopping tablet replica
I20261120 03:01:37.039378  3118 raft_consensus.cc:2440] T 45460f5605c844d0a28e598f42fdc4cb P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:37.039888  3118 raft_consensus.cc:2469] T 45460f5605c844d0a28e598f42fdc4cb P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:37.045727  3118 ts_tablet_manager.cc:1962] T 45460f5605c844d0a28e598f42fdc4cb P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:37.046859  3120 ts_tablet_manager.cc:1975] T 45460f5605c844d0a28e598f42fdc4cb P 23741ccd042d4ca49afc3b8d1e26e22c: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:37.051285  3114 tablet_bootstrap.cc:654] T f8087040bf3349e1898a858546ff1f04 P c393ca4a78704704bc9b58916a81fc3f: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:37.052044  3120 log.cc:1199] T 45460f5605c844d0a28e598f42fdc4cb P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/45460f5605c844d0a28e598f42fdc4cb
I20261120 03:01:37.060706  3120 ts_tablet_manager.cc:1996] T 45460f5605c844d0a28e598f42fdc4cb P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting consensus metadata
I20261120 03:01:37.073194  2495 catalog_manager.cc:5081] TS 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): tablet 45460f5605c844d0a28e598f42fdc4cb (table default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]) successfully deleted
I20261120 03:01:37.076905  2695 tablet_service.cc:1495] Processing CreateTablet for tablet 4858d6a8264141cf805794882d4f0d56 (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:37.078243  2695 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4858d6a8264141cf805794882d4f0d56. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:37.082511  3120 tablet_replica.cc:333] T 7c1a821b612b4325bc34a3a16c284b2d P 23741ccd042d4ca49afc3b8d1e26e22c: stopping tablet replica
I20261120 03:01:37.083240  3120 raft_consensus.cc:2440] T 7c1a821b612b4325bc34a3a16c284b2d P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:37.089313  3120 raft_consensus.cc:2469] T 7c1a821b612b4325bc34a3a16c284b2d P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:37.090034  2985 tablet_service.cc:1495] Processing CreateTablet for tablet 4858d6a8264141cf805794882d4f0d56 (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:37.091306  2985 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4858d6a8264141cf805794882d4f0d56. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:37.093163  3119 ts_tablet_manager.cc:1975] T 45460f5605c844d0a28e598f42fdc4cb P c393ca4a78704704bc9b58916a81fc3f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:37.096626  3120 ts_tablet_manager.cc:1962] T 7c1a821b612b4325bc34a3a16c284b2d P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting tablet data with delete state TABLET_DATA_DELETED
W20261120 03:01:37.099833  2913 tablet.cc:2554] T f8087040bf3349e1898a858546ff1f04 P 23741ccd042d4ca49afc3b8d1e26e22c: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20261120 03:01:37.102309  3114 tablet_bootstrap.cc:492] T f8087040bf3349e1898a858546ff1f04 P c393ca4a78704704bc9b58916a81fc3f: No bootstrap required, opened a new log
I20261120 03:01:37.102578  2505 master_service.cc:965] ReplaceTablet: received request to replace tablet f8087040bf3349e1898a858546ff1f04 from {username='slave'} at 127.0.0.1:57466
I20261120 03:01:37.102819  3114 ts_tablet_manager.cc:1449] T f8087040bf3349e1898a858546ff1f04 P c393ca4a78704704bc9b58916a81fc3f: Time spent bootstrapping tablet: real 0.087s	user 0.005s	sys 0.007s
I20261120 03:01:37.112375  2841 tablet_service.cc:1495] Processing CreateTablet for tablet 4858d6a8264141cf805794882d4f0d56 (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:37.113475  2841 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 4858d6a8264141cf805794882d4f0d56. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:37.128526  3114 raft_consensus.cc:366] T f8087040bf3349e1898a858546ff1f04 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.129244  3114 raft_consensus.cc:392] T f8087040bf3349e1898a858546ff1f04 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:37.129499  3114 raft_consensus.cc:787] T f8087040bf3349e1898a858546ff1f04 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c393ca4a78704704bc9b58916a81fc3f, State: Initialized, Role: FOLLOWER
I20261120 03:01:37.130370  3114 consensus_queue.cc:261] T f8087040bf3349e1898a858546ff1f04 P c393ca4a78704704bc9b58916a81fc3f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.132313  3114 ts_tablet_manager.cc:1480] T f8087040bf3349e1898a858546ff1f04 P c393ca4a78704704bc9b58916a81fc3f: Time spent starting tablet: real 0.029s	user 0.005s	sys 0.000s
I20261120 03:01:37.133244  3114 tablet_bootstrap.cc:492] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f: Bootstrap starting.
I20261120 03:01:37.140183  3119 log.cc:1199] T 45460f5605c844d0a28e598f42fdc4cb P c393ca4a78704704bc9b58916a81fc3f: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/45460f5605c844d0a28e598f42fdc4cb
I20261120 03:01:37.141151  3119 ts_tablet_manager.cc:1996] T 45460f5605c844d0a28e598f42fdc4cb P c393ca4a78704704bc9b58916a81fc3f: Deleting consensus metadata
I20261120 03:01:37.158327  2497 catalog_manager.cc:5081] TS c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993): tablet 45460f5605c844d0a28e598f42fdc4cb (table default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]) successfully deleted
I20261120 03:01:37.162153  3112 tablet_bootstrap.cc:492] T 4858d6a8264141cf805794882d4f0d56 P 23741ccd042d4ca49afc3b8d1e26e22c: Bootstrap starting.
I20261120 03:01:37.166062  3114 tablet_bootstrap.cc:654] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:37.168351  3112 tablet_bootstrap.cc:654] T 4858d6a8264141cf805794882d4f0d56 P 23741ccd042d4ca49afc3b8d1e26e22c: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:37.169662  2985 tablet_service.cc:1542] Processing DeleteTablet for tablet f8087040bf3349e1898a858546ff1f04 with delete_type TABLET_DATA_DELETED (replaced by tablet 1f017d7ebf3644e181aa529744601b52) from {username='slave'} at 127.0.0.1:41990
I20261120 03:01:37.170497  3119 tablet_replica.cc:333] T f8087040bf3349e1898a858546ff1f04 P c393ca4a78704704bc9b58916a81fc3f: stopping tablet replica
I20261120 03:01:37.170324  2505 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: Sending DeleteTablet for 3 replicas of tablet f8087040bf3349e1898a858546ff1f04
I20261120 03:01:37.171181  3119 raft_consensus.cc:2440] T f8087040bf3349e1898a858546ff1f04 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:37.171700  3119 raft_consensus.cc:2469] T f8087040bf3349e1898a858546ff1f04 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:37.173904  2694 tablet_service.cc:1542] Processing DeleteTablet for tablet f8087040bf3349e1898a858546ff1f04 with delete_type TABLET_DATA_DELETED (replaced by tablet 1f017d7ebf3644e181aa529744601b52) from {username='slave'} at 127.0.0.1:38150
I20261120 03:01:37.174461  3119 ts_tablet_manager.cc:1962] T f8087040bf3349e1898a858546ff1f04 P c393ca4a78704704bc9b58916a81fc3f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:37.179231  2841 tablet_service.cc:1542] Processing DeleteTablet for tablet f8087040bf3349e1898a858546ff1f04 with delete_type TABLET_DATA_DELETED (replaced by tablet 1f017d7ebf3644e181aa529744601b52) from {username='slave'} at 127.0.0.1:35818
I20261120 03:01:37.186770  3112 tablet_bootstrap.cc:492] T 4858d6a8264141cf805794882d4f0d56 P 23741ccd042d4ca49afc3b8d1e26e22c: No bootstrap required, opened a new log
I20261120 03:01:37.187073  3120 ts_tablet_manager.cc:1975] T 7c1a821b612b4325bc34a3a16c284b2d P 23741ccd042d4ca49afc3b8d1e26e22c: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:37.187222  3112 ts_tablet_manager.cc:1449] T 4858d6a8264141cf805794882d4f0d56 P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent bootstrapping tablet: real 0.025s	user 0.013s	sys 0.000s
I20261120 03:01:37.187430  3120 log.cc:1199] T 7c1a821b612b4325bc34a3a16c284b2d P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/7c1a821b612b4325bc34a3a16c284b2d
I20261120 03:01:37.188033  3120 ts_tablet_manager.cc:1996] T 7c1a821b612b4325bc34a3a16c284b2d P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting consensus metadata
I20261120 03:01:37.189998  3112 raft_consensus.cc:366] T 4858d6a8264141cf805794882d4f0d56 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } }
I20261120 03:01:37.190683  3112 raft_consensus.cc:392] T 4858d6a8264141cf805794882d4f0d56 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:37.190944  3112 raft_consensus.cc:787] T 4858d6a8264141cf805794882d4f0d56 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23741ccd042d4ca49afc3b8d1e26e22c, State: Initialized, Role: FOLLOWER
I20261120 03:01:37.193158  3120 tablet_replica.cc:333] T f8087040bf3349e1898a858546ff1f04 P 23741ccd042d4ca49afc3b8d1e26e22c: stopping tablet replica
I20261120 03:01:37.193116  2495 catalog_manager.cc:5081] TS 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): tablet 7c1a821b612b4325bc34a3a16c284b2d (table default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]) successfully deleted
I20261120 03:01:37.191865  3112 consensus_queue.cc:261] T 4858d6a8264141cf805794882d4f0d56 P 23741ccd042d4ca49afc3b8d1e26e22c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } }
I20261120 03:01:37.193815  3120 raft_consensus.cc:2440] T f8087040bf3349e1898a858546ff1f04 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:37.196864  3112 ts_tablet_manager.cc:1480] T 4858d6a8264141cf805794882d4f0d56 P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent starting tablet: real 0.009s	user 0.006s	sys 0.000s
I20261120 03:01:37.203660  2503 master_service.cc:965] ReplaceTablet: received request to replace tablet f8087040bf3349e1898a858546ff1f04 from {username='slave'} at 127.0.0.1:57466
I20261120 03:01:37.211699  2985 tablet_service.cc:1542] Processing DeleteTablet for tablet f8087040bf3349e1898a858546ff1f04 with delete_type TABLET_DATA_DELETED (replaced by tablet 1f017d7ebf3644e181aa529744601b52) from {username='slave'} at 127.0.0.1:41990
I20261120 03:01:37.212548  2505 catalog_manager.cc:6952] ReplaceTablet: tablet f8087040bf3349e1898a858546ff1f04 deleted and replaced by tablet 1f017d7ebf3644e181aa529744601b52
I20261120 03:01:37.218679  3120 raft_consensus.cc:2469] T f8087040bf3349e1898a858546ff1f04 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:37.222390  3113 tablet_bootstrap.cc:492] T 7c1a821b612b4325bc34a3a16c284b2d P d9ed1d903ef340ac8f1025c2b7df9bdd: No bootstrap required, opened a new log
I20261120 03:01:37.222856  3113 ts_tablet_manager.cc:1449] T 7c1a821b612b4325bc34a3a16c284b2d P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent bootstrapping tablet: real 0.196s	user 0.023s	sys 0.027s
I20261120 03:01:37.225076  3118 ts_tablet_manager.cc:1975] T 45460f5605c844d0a28e598f42fdc4cb P d9ed1d903ef340ac8f1025c2b7df9bdd: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:37.225582  3114 tablet_bootstrap.cc:492] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f: No bootstrap required, opened a new log
I20261120 03:01:37.226050  3114 ts_tablet_manager.cc:1449] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f: Time spent bootstrapping tablet: real 0.093s	user 0.011s	sys 0.000s
I20261120 03:01:37.228917  3114 raft_consensus.cc:366] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.229549  3114 raft_consensus.cc:392] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:37.229825  3114 raft_consensus.cc:787] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c393ca4a78704704bc9b58916a81fc3f, State: Initialized, Role: FOLLOWER
I20261120 03:01:37.232362  3114 consensus_queue.cc:261] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.233299  3120 ts_tablet_manager.cc:1962] T f8087040bf3349e1898a858546ff1f04 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:37.240415  3114 ts_tablet_manager.cc:1480] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f: Time spent starting tablet: real 0.014s	user 0.004s	sys 0.002s
I20261120 03:01:37.231572  3119 ts_tablet_manager.cc:1975] T f8087040bf3349e1898a858546ff1f04 P c393ca4a78704704bc9b58916a81fc3f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:37.240993  3119 log.cc:1199] T f8087040bf3349e1898a858546ff1f04 P c393ca4a78704704bc9b58916a81fc3f: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/f8087040bf3349e1898a858546ff1f04
I20261120 03:01:37.241652  3119 ts_tablet_manager.cc:1996] T f8087040bf3349e1898a858546ff1f04 P c393ca4a78704704bc9b58916a81fc3f: Deleting consensus metadata
I20261120 03:01:37.244208  2497 catalog_manager.cc:5081] TS c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993): tablet f8087040bf3349e1898a858546ff1f04 (table default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]) successfully deleted
W20261120 03:01:37.248283  2505 catalog_manager.cc:5689] Ignoring report from unknown tablet 7c1a821b612b4325bc34a3a16c284b2d
I20261120 03:01:37.252686  3114 tablet_bootstrap.cc:492] T 4858d6a8264141cf805794882d4f0d56 P c393ca4a78704704bc9b58916a81fc3f: Bootstrap starting.
W20261120 03:01:37.253266  2502 catalog_manager.cc:5689] Ignoring report from unknown tablet 7c1a821b612b4325bc34a3a16c284b2d
W20261120 03:01:37.255188  2497 catalog_manager.cc:5048] TS c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993): delete failed for tablet f8087040bf3349e1898a858546ff1f04 because the tablet was not found. No further retry: Not found: Tablet not found: f8087040bf3349e1898a858546ff1f04
I20261120 03:01:37.225606  3113 raft_consensus.cc:366] T 7c1a821b612b4325bc34a3a16c284b2d P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.262187  3113 raft_consensus.cc:392] T 7c1a821b612b4325bc34a3a16c284b2d P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:37.262341  3118 log.cc:1199] T 45460f5605c844d0a28e598f42fdc4cb P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/45460f5605c844d0a28e598f42fdc4cb
I20261120 03:01:37.262517  3113 raft_consensus.cc:787] T 7c1a821b612b4325bc34a3a16c284b2d P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9ed1d903ef340ac8f1025c2b7df9bdd, State: Initialized, Role: FOLLOWER
I20261120 03:01:37.263463  3118 ts_tablet_manager.cc:1996] T 45460f5605c844d0a28e598f42fdc4cb P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting consensus metadata
I20261120 03:01:37.263236  2695 tablet_service.cc:1495] Processing CreateTablet for tablet 1f017d7ebf3644e181aa529744601b52 (DEFAULT_TABLE table=default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:37.263195  3113 consensus_queue.cc:261] T 7c1a821b612b4325bc34a3a16c284b2d P d9ed1d903ef340ac8f1025c2b7df9bdd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.266300  2841 tablet_service.cc:1495] Processing CreateTablet for tablet 1f017d7ebf3644e181aa529744601b52 (DEFAULT_TABLE table=default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:37.266798  3113 ts_tablet_manager.cc:1480] T 7c1a821b612b4325bc34a3a16c284b2d P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent starting tablet: real 0.044s	user 0.005s	sys 0.002s
I20261120 03:01:37.267433  2841 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1f017d7ebf3644e181aa529744601b52. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:37.268180  3113 tablet_bootstrap.cc:492] T f8087040bf3349e1898a858546ff1f04 P d9ed1d903ef340ac8f1025c2b7df9bdd: Bootstrap starting.
W20261120 03:01:37.270673  2503 catalog_manager.cc:5689] Ignoring report from unknown tablet 7c1a821b612b4325bc34a3a16c284b2d
I20261120 03:01:37.273903  2695 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1f017d7ebf3644e181aa529744601b52. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:37.274004  2985 tablet_service.cc:1495] Processing CreateTablet for tablet 1f017d7ebf3644e181aa529744601b52 (DEFAULT_TABLE table=default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:37.275228  2985 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1f017d7ebf3644e181aa529744601b52. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:37.275264  3120 ts_tablet_manager.cc:1975] T f8087040bf3349e1898a858546ff1f04 P 23741ccd042d4ca49afc3b8d1e26e22c: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:37.275655  3120 log.cc:1199] T f8087040bf3349e1898a858546ff1f04 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/f8087040bf3349e1898a858546ff1f04
I20261120 03:01:37.276342  3120 ts_tablet_manager.cc:1996] T f8087040bf3349e1898a858546ff1f04 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting consensus metadata
I20261120 03:01:37.279840  2495 catalog_manager.cc:5081] TS 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): tablet f8087040bf3349e1898a858546ff1f04 (table default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]) successfully deleted
I20261120 03:01:37.280385  2497 catalog_manager.cc:5081] TS d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769): tablet 45460f5605c844d0a28e598f42fdc4cb (table default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]) successfully deleted
W20261120 03:01:37.281558  2497 catalog_manager.cc:5061] TS d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769): delete failed for tablet f8087040bf3349e1898a858546ff1f04 because tablet deleting was already in progress. No further retry: Already present: State transition of tablet f8087040bf3349e1898a858546ff1f04 already in progress: creating tablet
I20261120 03:01:37.283150  3113 tablet_bootstrap.cc:654] T f8087040bf3349e1898a858546ff1f04 P d9ed1d903ef340ac8f1025c2b7df9bdd: Neither blocks nor log segments found. Creating new log.
W20261120 03:01:37.285079  2503 catalog_manager.cc:5689] Ignoring report from unknown tablet 7c1a821b612b4325bc34a3a16c284b2d
W20261120 03:01:37.295089  2769 tablet.cc:2554] T 7c1a821b612b4325bc34a3a16c284b2d P d9ed1d903ef340ac8f1025c2b7df9bdd: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20261120 03:01:37.302498  3114 tablet_bootstrap.cc:654] T 4858d6a8264141cf805794882d4f0d56 P c393ca4a78704704bc9b58916a81fc3f: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:37.313253  3112 tablet_bootstrap.cc:492] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c: Bootstrap starting.
I20261120 03:01:37.315553  3113 tablet_bootstrap.cc:492] T f8087040bf3349e1898a858546ff1f04 P d9ed1d903ef340ac8f1025c2b7df9bdd: No bootstrap required, opened a new log
I20261120 03:01:37.315999  3113 ts_tablet_manager.cc:1449] T f8087040bf3349e1898a858546ff1f04 P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent bootstrapping tablet: real 0.048s	user 0.004s	sys 0.006s
I20261120 03:01:37.319568  3112 tablet_bootstrap.cc:654] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:37.319906  3113 raft_consensus.cc:366] T f8087040bf3349e1898a858546ff1f04 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.320628  3113 raft_consensus.cc:392] T f8087040bf3349e1898a858546ff1f04 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:37.321327  3113 raft_consensus.cc:787] T f8087040bf3349e1898a858546ff1f04 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9ed1d903ef340ac8f1025c2b7df9bdd, State: Initialized, Role: FOLLOWER
I20261120 03:01:37.323557  3114 tablet_bootstrap.cc:492] T 4858d6a8264141cf805794882d4f0d56 P c393ca4a78704704bc9b58916a81fc3f: No bootstrap required, opened a new log
I20261120 03:01:37.324018  3114 ts_tablet_manager.cc:1449] T 4858d6a8264141cf805794882d4f0d56 P c393ca4a78704704bc9b58916a81fc3f: Time spent bootstrapping tablet: real 0.072s	user 0.012s	sys 0.000s
I20261120 03:01:37.326596  3114 raft_consensus.cc:366] T 4858d6a8264141cf805794882d4f0d56 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } }
I20261120 03:01:37.327307  3114 raft_consensus.cc:392] T 4858d6a8264141cf805794882d4f0d56 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:37.327567  3114 raft_consensus.cc:787] T 4858d6a8264141cf805794882d4f0d56 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c393ca4a78704704bc9b58916a81fc3f, State: Initialized, Role: FOLLOWER
I20261120 03:01:37.328370  3114 consensus_queue.cc:261] T 4858d6a8264141cf805794882d4f0d56 P c393ca4a78704704bc9b58916a81fc3f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } }
I20261120 03:01:37.342316  3113 consensus_queue.cc:261] T f8087040bf3349e1898a858546ff1f04 P d9ed1d903ef340ac8f1025c2b7df9bdd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.345093  3114 ts_tablet_manager.cc:1480] T 4858d6a8264141cf805794882d4f0d56 P c393ca4a78704704bc9b58916a81fc3f: Time spent starting tablet: real 0.021s	user 0.005s	sys 0.001s
I20261120 03:01:37.346096  3112 tablet_bootstrap.cc:492] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c: No bootstrap required, opened a new log
I20261120 03:01:37.346534  3112 ts_tablet_manager.cc:1449] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent bootstrapping tablet: real 0.033s	user 0.008s	sys 0.006s
I20261120 03:01:37.352691  3112 raft_consensus.cc:366] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.353391  3112 raft_consensus.cc:392] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:37.353593  3113 ts_tablet_manager.cc:1480] T f8087040bf3349e1898a858546ff1f04 P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent starting tablet: real 0.037s	user 0.004s	sys 0.003s
I20261120 03:01:37.353662  3112 raft_consensus.cc:787] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23741ccd042d4ca49afc3b8d1e26e22c, State: Initialized, Role: FOLLOWER
I20261120 03:01:37.357425  2695 tablet_service.cc:1542] Processing DeleteTablet for tablet f8087040bf3349e1898a858546ff1f04 with delete_type TABLET_DATA_DELETED (replaced by tablet 1f017d7ebf3644e181aa529744601b52) from {username='slave'} at 127.0.0.1:38150
I20261120 03:01:37.358206  3118 tablet_replica.cc:333] T f8087040bf3349e1898a858546ff1f04 P d9ed1d903ef340ac8f1025c2b7df9bdd: stopping tablet replica
I20261120 03:01:37.358384  3112 consensus_queue.cc:261] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.358911  3118 raft_consensus.cc:2440] T f8087040bf3349e1898a858546ff1f04 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:37.359359  3118 raft_consensus.cc:2469] T f8087040bf3349e1898a858546ff1f04 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:37.365725  3118 ts_tablet_manager.cc:1962] T f8087040bf3349e1898a858546ff1f04 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:37.371248  3112 ts_tablet_manager.cc:1480] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent starting tablet: real 0.021s	user 0.005s	sys 0.000s
I20261120 03:01:37.372104  3114 tablet_bootstrap.cc:492] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f: Bootstrap starting.
I20261120 03:01:37.383529  3114 tablet_bootstrap.cc:654] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:37.395664  3113 tablet_bootstrap.cc:492] T 4858d6a8264141cf805794882d4f0d56 P d9ed1d903ef340ac8f1025c2b7df9bdd: Bootstrap starting.
I20261120 03:01:37.401701  3118 ts_tablet_manager.cc:1975] T f8087040bf3349e1898a858546ff1f04 P d9ed1d903ef340ac8f1025c2b7df9bdd: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:37.402012  3113 tablet_bootstrap.cc:654] T 4858d6a8264141cf805794882d4f0d56 P d9ed1d903ef340ac8f1025c2b7df9bdd: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:37.402139  3118 log.cc:1199] T f8087040bf3349e1898a858546ff1f04 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/f8087040bf3349e1898a858546ff1f04
I20261120 03:01:37.402786  3118 ts_tablet_manager.cc:1996] T f8087040bf3349e1898a858546ff1f04 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting consensus metadata
I20261120 03:01:37.403822  3123 raft_consensus.cc:534] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:37.404466  3123 raft_consensus.cc:556] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.405784  2497 catalog_manager.cc:5081] TS d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769): tablet f8087040bf3349e1898a858546ff1f04 (table default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]) successfully deleted
I20261120 03:01:37.406353  3114 tablet_bootstrap.cc:492] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f: No bootstrap required, opened a new log
I20261120 03:01:37.406908  3114 ts_tablet_manager.cc:1449] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f: Time spent bootstrapping tablet: real 0.035s	user 0.013s	sys 0.005s
I20261120 03:01:37.409703  3114 raft_consensus.cc:366] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.410386  3114 raft_consensus.cc:392] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:37.410660  3114 raft_consensus.cc:787] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c393ca4a78704704bc9b58916a81fc3f, State: Initialized, Role: FOLLOWER
I20261120 03:01:37.411495  3114 consensus_queue.cc:261] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.414507  2502 master_service.cc:965] ReplaceTablet: received request to replace tablet 7c1a821b612b4325bc34a3a16c284b2d from {username='slave'} at 127.0.0.1:57466
I20261120 03:01:37.422500  3114 ts_tablet_manager.cc:1480] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f: Time spent starting tablet: real 0.015s	user 0.006s	sys 0.000s
I20261120 03:01:37.432834  3123 leader_election.cc:290] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769), c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993)
I20261120 03:01:37.437358  3113 tablet_bootstrap.cc:492] T 4858d6a8264141cf805794882d4f0d56 P d9ed1d903ef340ac8f1025c2b7df9bdd: No bootstrap required, opened a new log
I20261120 03:01:37.437783  3113 ts_tablet_manager.cc:1449] T 4858d6a8264141cf805794882d4f0d56 P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent bootstrapping tablet: real 0.042s	user 0.009s	sys 0.000s
I20261120 03:01:37.440677  3113 raft_consensus.cc:366] T 4858d6a8264141cf805794882d4f0d56 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } }
I20261120 03:01:37.441345  3113 raft_consensus.cc:392] T 4858d6a8264141cf805794882d4f0d56 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:37.441625  3113 raft_consensus.cc:787] T 4858d6a8264141cf805794882d4f0d56 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9ed1d903ef340ac8f1025c2b7df9bdd, State: Initialized, Role: FOLLOWER
I20261120 03:01:37.442315  3113 consensus_queue.cc:261] T 4858d6a8264141cf805794882d4f0d56 P d9ed1d903ef340ac8f1025c2b7df9bdd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } }
I20261120 03:01:37.450774  3113 ts_tablet_manager.cc:1480] T 4858d6a8264141cf805794882d4f0d56 P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent starting tablet: real 0.013s	user 0.005s	sys 0.004s
I20261120 03:01:37.462442  3113 tablet_bootstrap.cc:492] T 1f017d7ebf3644e181aa529744601b52 P d9ed1d903ef340ac8f1025c2b7df9bdd: Bootstrap starting.
I20261120 03:01:37.462615  2502 master_service.cc:965] ReplaceTablet: received request to replace tablet 4858d6a8264141cf805794882d4f0d56 from {username='slave'} at 127.0.0.1:57466
I20261120 03:01:37.469012  3113 tablet_bootstrap.cc:654] T 1f017d7ebf3644e181aa529744601b52 P d9ed1d903ef340ac8f1025c2b7df9bdd: Neither blocks nor log segments found. Creating new log.
W20261120 03:01:37.478161  3053 tablet.cc:2554] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20261120 03:01:37.481709  3005 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1f017d7ebf3644e181aa529744601b52" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c393ca4a78704704bc9b58916a81fc3f" is_pre_election: true
I20261120 03:01:37.482864  3005 raft_consensus.cc:2062] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 23741ccd042d4ca49afc3b8d1e26e22c in term 0
I20261120 03:01:37.483485  2715 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1f017d7ebf3644e181aa529744601b52" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" is_pre_election: true
I20261120 03:01:37.485570  2799 leader_election.cc:304] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 23741ccd042d4ca49afc3b8d1e26e22c, c393ca4a78704704bc9b58916a81fc3f; no voters: 
I20261120 03:01:37.486593  3123 raft_consensus.cc:3024] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:37.486958  3123 raft_consensus.cc:534] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:37.487286  3123 raft_consensus.cc:3296] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:37.486696  2502 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: Sending DeleteTablet for 3 replicas of tablet 4858d6a8264141cf805794882d4f0d56
W20261120 03:01:37.491752  2799 leader_election.cc:343] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:37.492998  2695 tablet_service.cc:1542] Processing DeleteTablet for tablet 4858d6a8264141cf805794882d4f0d56 with delete_type TABLET_DATA_DELETED (replaced by tablet 664b7403286d480788691bddb6816965) from {username='slave'} at 127.0.0.1:38150
I20261120 03:01:37.494472  3118 tablet_replica.cc:333] T 4858d6a8264141cf805794882d4f0d56 P d9ed1d903ef340ac8f1025c2b7df9bdd: stopping tablet replica
I20261120 03:01:37.495208  3118 raft_consensus.cc:2440] T 4858d6a8264141cf805794882d4f0d56 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:37.495435  2502 catalog_manager.cc:6952] ReplaceTablet: tablet 4858d6a8264141cf805794882d4f0d56 deleted and replaced by tablet 664b7403286d480788691bddb6816965
I20261120 03:01:37.495676  3118 raft_consensus.cc:2469] T 4858d6a8264141cf805794882d4f0d56 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:37.496412  2985 tablet_service.cc:1542] Processing DeleteTablet for tablet 4858d6a8264141cf805794882d4f0d56 with delete_type TABLET_DATA_DELETED (replaced by tablet 664b7403286d480788691bddb6816965) from {username='slave'} at 127.0.0.1:41990
I20261120 03:01:37.499270  2841 tablet_service.cc:1542] Processing DeleteTablet for tablet 4858d6a8264141cf805794882d4f0d56 with delete_type TABLET_DATA_DELETED (replaced by tablet 664b7403286d480788691bddb6816965) from {username='slave'} at 127.0.0.1:35818
I20261120 03:01:37.500039  3120 tablet_replica.cc:333] T 4858d6a8264141cf805794882d4f0d56 P 23741ccd042d4ca49afc3b8d1e26e22c: stopping tablet replica
I20261120 03:01:37.500787  3120 raft_consensus.cc:2440] T 4858d6a8264141cf805794882d4f0d56 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:37.501272  3120 raft_consensus.cc:2469] T 4858d6a8264141cf805794882d4f0d56 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:37.497687  3119 tablet_replica.cc:333] T 4858d6a8264141cf805794882d4f0d56 P c393ca4a78704704bc9b58916a81fc3f: stopping tablet replica
I20261120 03:01:37.502660  3119 raft_consensus.cc:2440] T 4858d6a8264141cf805794882d4f0d56 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:37.503156  3119 raft_consensus.cc:2469] T 4858d6a8264141cf805794882d4f0d56 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:37.503793  3120 ts_tablet_manager.cc:1962] T 4858d6a8264141cf805794882d4f0d56 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:37.507033  3119 ts_tablet_manager.cc:1962] T 4858d6a8264141cf805794882d4f0d56 P c393ca4a78704704bc9b58916a81fc3f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:37.505422  3118 ts_tablet_manager.cc:1962] T 4858d6a8264141cf805794882d4f0d56 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:37.522508  2695 tablet_service.cc:1495] Processing CreateTablet for tablet 664b7403286d480788691bddb6816965 (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:37.523693  2695 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 664b7403286d480788691bddb6816965. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:37.528000  2985 tablet_service.cc:1495] Processing CreateTablet for tablet 664b7403286d480788691bddb6816965 (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:37.529122  2985 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 664b7403286d480788691bddb6816965. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:37.531378  2841 tablet_service.cc:1495] Processing CreateTablet for tablet 664b7403286d480788691bddb6816965 (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:37.532522  2841 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 664b7403286d480788691bddb6816965. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:37.535395  3123 raft_consensus.cc:556] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.542184  3120 ts_tablet_manager.cc:1975] T 4858d6a8264141cf805794882d4f0d56 P 23741ccd042d4ca49afc3b8d1e26e22c: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:37.542579  3120 log.cc:1199] T 4858d6a8264141cf805794882d4f0d56 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/4858d6a8264141cf805794882d4f0d56
I20261120 03:01:37.543251  3120 ts_tablet_manager.cc:1996] T 4858d6a8264141cf805794882d4f0d56 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting consensus metadata
I20261120 03:01:37.546243  2495 catalog_manager.cc:5081] TS 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): tablet 4858d6a8264141cf805794882d4f0d56 (table default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]) successfully deleted
I20261120 03:01:37.553041  3123 leader_election.cc:290] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 election: Requested vote from peers d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769), c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993)
I20261120 03:01:37.555043  2715 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1f017d7ebf3644e181aa529744601b52" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd"
I20261120 03:01:37.557293  3005 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1f017d7ebf3644e181aa529744601b52" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c393ca4a78704704bc9b58916a81fc3f"
I20261120 03:01:37.557920  3005 raft_consensus.cc:3296] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:37.559092  3005 raft_consensus.cc:2062] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 23741ccd042d4ca49afc3b8d1e26e22c in term 1
W20261120 03:01:37.562848  2799 leader_election.cc:343] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:37.571027  2799 leader_election.cc:304] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 23741ccd042d4ca49afc3b8d1e26e22c, c393ca4a78704704bc9b58916a81fc3f; no voters: d9ed1d903ef340ac8f1025c2b7df9bdd
I20261120 03:01:37.572144  3123 raft_consensus.cc:3024] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:37.583477  3113 tablet_bootstrap.cc:492] T 1f017d7ebf3644e181aa529744601b52 P d9ed1d903ef340ac8f1025c2b7df9bdd: No bootstrap required, opened a new log
I20261120 03:01:37.584131  3113 ts_tablet_manager.cc:1449] T 1f017d7ebf3644e181aa529744601b52 P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent bootstrapping tablet: real 0.122s	user 0.013s	sys 0.018s
I20261120 03:01:37.585131  3123 raft_consensus.cc:744] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 LEADER]: Becoming Leader. State: Replica: 23741ccd042d4ca49afc3b8d1e26e22c, State: Running, Role: LEADER
I20261120 03:01:37.586023  3123 consensus_queue.cc:238] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.588187  3113 raft_consensus.cc:366] T 1f017d7ebf3644e181aa529744601b52 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.599445  3113 raft_consensus.cc:392] T 1f017d7ebf3644e181aa529744601b52 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:37.600754  3113 raft_consensus.cc:787] T 1f017d7ebf3644e181aa529744601b52 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9ed1d903ef340ac8f1025c2b7df9bdd, State: Initialized, Role: FOLLOWER
I20261120 03:01:37.601909  3112 tablet_bootstrap.cc:492] T 664b7403286d480788691bddb6816965 P 23741ccd042d4ca49afc3b8d1e26e22c: Bootstrap starting.
I20261120 03:01:37.611662  3112 tablet_bootstrap.cc:654] T 664b7403286d480788691bddb6816965 P 23741ccd042d4ca49afc3b8d1e26e22c: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:37.615145  3114 tablet_bootstrap.cc:492] T 664b7403286d480788691bddb6816965 P c393ca4a78704704bc9b58916a81fc3f: Bootstrap starting.
I20261120 03:01:37.615967  3113 consensus_queue.cc:261] T 1f017d7ebf3644e181aa529744601b52 P d9ed1d903ef340ac8f1025c2b7df9bdd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.621474  3114 tablet_bootstrap.cc:654] T 664b7403286d480788691bddb6816965 P c393ca4a78704704bc9b58916a81fc3f: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:37.593537  3118 ts_tablet_manager.cc:1975] T 4858d6a8264141cf805794882d4f0d56 P d9ed1d903ef340ac8f1025c2b7df9bdd: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:37.622206  3118 log.cc:1199] T 4858d6a8264141cf805794882d4f0d56 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/4858d6a8264141cf805794882d4f0d56
I20261120 03:01:37.625566  3119 ts_tablet_manager.cc:1975] T 4858d6a8264141cf805794882d4f0d56 P c393ca4a78704704bc9b58916a81fc3f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:37.628165  3118 ts_tablet_manager.cc:1996] T 4858d6a8264141cf805794882d4f0d56 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting consensus metadata
I20261120 03:01:37.625978  3119 log.cc:1199] T 4858d6a8264141cf805794882d4f0d56 P c393ca4a78704704bc9b58916a81fc3f: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/4858d6a8264141cf805794882d4f0d56
I20261120 03:01:37.630293  3119 ts_tablet_manager.cc:1996] T 4858d6a8264141cf805794882d4f0d56 P c393ca4a78704704bc9b58916a81fc3f: Deleting consensus metadata
I20261120 03:01:37.627430  3113 ts_tablet_manager.cc:1480] T 1f017d7ebf3644e181aa529744601b52 P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent starting tablet: real 0.043s	user 0.006s	sys 0.000s
I20261120 03:01:37.632450  2502 master_service.cc:965] ReplaceTablet: received request to replace tablet 4858d6a8264141cf805794882d4f0d56 from {username='slave'} at 127.0.0.1:57466
I20261120 03:01:37.634821  2497 catalog_manager.cc:5081] TS c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993): tablet 4858d6a8264141cf805794882d4f0d56 (table default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]) successfully deleted
I20261120 03:01:37.638089  3113 tablet_bootstrap.cc:492] T 664b7403286d480788691bddb6816965 P d9ed1d903ef340ac8f1025c2b7df9bdd: Bootstrap starting.
I20261120 03:01:37.646106  2502 catalog_manager.cc:5856] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c reported cstate change: term changed from 0 to 1, leader changed from <none> to 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66). New cstate: current_term: 1 leader_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } health_report { overall_health: UNKNOWN } } }
I20261120 03:01:37.653836  3144 consensus_queue.cc:1049] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20261120 03:01:37.664465  2497 catalog_manager.cc:5081] TS d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769): tablet 4858d6a8264141cf805794882d4f0d56 (table default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]) successfully deleted
I20261120 03:01:37.671468  2715 raft_consensus.cc:3296] T 1f017d7ebf3644e181aa529744601b52 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Advancing to term 1
03:01:37.670 [HMS - ERROR - pool-6-thread-1] (RetryingHMSHandler.java:201) AlreadyExistsException(message:Table hive.default.table_ad2a1b8b11fe4111b57b8eeae744cf96 already exists)
	at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.create_table_core(HiveMetaStore.java:1839)
	at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.create_table_core(HiveMetaStore.java:1786)
	at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.create_table_with_environment_context(HiveMetaStore.java:2035)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:569)
	at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:147)
	at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:108)
	at jdk.proxy2/jdk.proxy2.$Proxy27.create_table_with_environment_context(Unknown Source)
	at org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Processor$create_table_with_environment_context.getResult(ThriftHiveMetastore.java:14800)
	at org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Processor$create_table_with_environment_context.getResult(ThriftHiveMetastore.java:14784)
	at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
	at org.apache.hadoop.hive.metastore.TUGIBasedProcessor.process(TUGIBasedProcessor.java:104)
	at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:286)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	at java.base/java.lang.Thread.run(Thread.java:840)

I20261120 03:01:37.679553  3113 tablet_bootstrap.cc:654] T 664b7403286d480788691bddb6816965 P d9ed1d903ef340ac8f1025c2b7df9bdd: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:37.681351  3142 consensus_queue.cc:1049] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [LEADER]: Connected to new peer: Peer: permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
W20261120 03:01:37.684290  2564 hms_client.cc:335] Time spent create HMS table: real 1.042s	user 0.000s	sys 0.000s
W20261120 03:01:37.684994  2510 catalog_manager.cc:2214] Already present: failed to create HMS catalog entry for table  [id=78309e9823ed4368aac108fad1e8c230]: failed to create Hive MetaStore table: TException - service has thrown: AlreadyExistsException(message=Table hive.default.table_ad2a1b8b11fe4111b57b8eeae744cf96 already exists)
I20261120 03:01:37.686012  2510 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:57466 (request call id 14) took 1534 ms. Trace:
I20261120 03:01:37.686408  2510 rpcz_store.cc:276] 1120 03:01:36.151199 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:36.151371 (+   172us) service_pool.cc:224] Handling call
1120 03:01:36.166346 (+ 14975us) catalog_manager.cc:2145] Acquired catalog manager lock
1120 03:01:36.180720 (+ 14374us) catalog_manager.cc:2191] Created new table and tablet info
1120 03:01:37.685933 (+1505213us) inbound_call.cc:177] Queueing success response
Metrics: {"Hive Metastore.queue_time_us":461174,"Hive Metastore.run_cpu_time_us":2128,"Hive Metastore.run_wall_time_us":1042281}
I20261120 03:01:37.692221  2510 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:57466:
name: "default.Table_4eedec7d9c82426698cb181fa17e7adb"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:37.711599  3112 tablet_bootstrap.cc:492] T 664b7403286d480788691bddb6816965 P 23741ccd042d4ca49afc3b8d1e26e22c: No bootstrap required, opened a new log
I20261120 03:01:37.731977  3113 tablet_bootstrap.cc:492] T 664b7403286d480788691bddb6816965 P d9ed1d903ef340ac8f1025c2b7df9bdd: No bootstrap required, opened a new log
I20261120 03:01:37.732416  3113 ts_tablet_manager.cc:1449] T 664b7403286d480788691bddb6816965 P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent bootstrapping tablet: real 0.095s	user 0.010s	sys 0.001s
I20261120 03:01:37.735049  3113 raft_consensus.cc:366] T 664b7403286d480788691bddb6816965 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.735733  3113 raft_consensus.cc:392] T 664b7403286d480788691bddb6816965 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:37.712487  3112 ts_tablet_manager.cc:1449] T 664b7403286d480788691bddb6816965 P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent bootstrapping tablet: real 0.111s	user 0.010s	sys 0.000s
I20261120 03:01:37.745257  3112 raft_consensus.cc:366] T 664b7403286d480788691bddb6816965 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.747242  3114 tablet_bootstrap.cc:492] T 664b7403286d480788691bddb6816965 P c393ca4a78704704bc9b58916a81fc3f: No bootstrap required, opened a new log
I20261120 03:01:37.747824  3114 ts_tablet_manager.cc:1449] T 664b7403286d480788691bddb6816965 P c393ca4a78704704bc9b58916a81fc3f: Time spent bootstrapping tablet: real 0.133s	user 0.039s	sys 0.080s
I20261120 03:01:37.736024  3113 raft_consensus.cc:787] T 664b7403286d480788691bddb6816965 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9ed1d903ef340ac8f1025c2b7df9bdd, State: Initialized, Role: FOLLOWER
I20261120 03:01:37.750499  3114 raft_consensus.cc:366] T 664b7403286d480788691bddb6816965 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.751144  3114 raft_consensus.cc:392] T 664b7403286d480788691bddb6816965 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:37.751447  3114 raft_consensus.cc:787] T 664b7403286d480788691bddb6816965 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c393ca4a78704704bc9b58916a81fc3f, State: Initialized, Role: FOLLOWER
I20261120 03:01:37.752123  3113 consensus_queue.cc:261] T 664b7403286d480788691bddb6816965 P d9ed1d903ef340ac8f1025c2b7df9bdd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.752205  3114 consensus_queue.cc:261] T 664b7403286d480788691bddb6816965 P c393ca4a78704704bc9b58916a81fc3f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.754231  3113 ts_tablet_manager.cc:1480] T 664b7403286d480788691bddb6816965 P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent starting tablet: real 0.022s	user 0.003s	sys 0.005s
I20261120 03:01:37.757270  3114 ts_tablet_manager.cc:1480] T 664b7403286d480788691bddb6816965 P c393ca4a78704704bc9b58916a81fc3f: Time spent starting tablet: real 0.009s	user 0.005s	sys 0.000s
I20261120 03:01:37.745915  3112 raft_consensus.cc:392] T 664b7403286d480788691bddb6816965 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:37.854290  3112 raft_consensus.cc:787] T 664b7403286d480788691bddb6816965 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23741ccd042d4ca49afc3b8d1e26e22c, State: Initialized, Role: FOLLOWER
I20261120 03:01:37.860788  3112 consensus_queue.cc:261] T 664b7403286d480788691bddb6816965 P 23741ccd042d4ca49afc3b8d1e26e22c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.881358  2503 master_service.cc:965] ReplaceTablet: received request to replace tablet 664b7403286d480788691bddb6816965 from {username='slave'} at 127.0.0.1:57466
I20261120 03:01:37.881342  2505 master_service.cc:965] ReplaceTablet: received request to replace tablet 1f017d7ebf3644e181aa529744601b52 from {username='slave'} at 127.0.0.1:57466
I20261120 03:01:37.885411  3112 ts_tablet_manager.cc:1480] T 664b7403286d480788691bddb6816965 P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent starting tablet: real 0.143s	user 0.006s	sys 0.000s
I20261120 03:01:37.921597  2503 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: Sending DeleteTablet for 3 replicas of tablet 664b7403286d480788691bddb6816965
I20261120 03:01:37.921694  2695 tablet_service.cc:1542] Processing DeleteTablet for tablet 1f017d7ebf3644e181aa529744601b52 with delete_type TABLET_DATA_DELETED (replaced by tablet e61b8f296bad46e4bac632f7e784633b) from {username='slave'} at 127.0.0.1:38150
I20261120 03:01:37.923177  2503 catalog_manager.cc:6952] ReplaceTablet: tablet 664b7403286d480788691bddb6816965 deleted and replaced by tablet 1d00a31328c245a88512188ff5f3e584
I20261120 03:01:37.924125  3118 tablet_replica.cc:333] T 1f017d7ebf3644e181aa529744601b52 P d9ed1d903ef340ac8f1025c2b7df9bdd: stopping tablet replica
I20261120 03:01:37.924108  2505 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8: Sending DeleteTablet for 3 replicas of tablet 1f017d7ebf3644e181aa529744601b52
I20261120 03:01:37.924814  3118 raft_consensus.cc:2440] T 1f017d7ebf3644e181aa529744601b52 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:37.925377  3118 raft_consensus.cc:2469] T 1f017d7ebf3644e181aa529744601b52 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:37.926271  2505 catalog_manager.cc:6952] ReplaceTablet: tablet 1f017d7ebf3644e181aa529744601b52 deleted and replaced by tablet e61b8f296bad46e4bac632f7e784633b
I20261120 03:01:37.927819  2841 tablet_service.cc:1542] Processing DeleteTablet for tablet 664b7403286d480788691bddb6816965 with delete_type TABLET_DATA_DELETED (replaced by tablet 1d00a31328c245a88512188ff5f3e584) from {username='slave'} at 127.0.0.1:35818
I20261120 03:01:37.928504  2842 tablet_service.cc:1542] Processing DeleteTablet for tablet 1f017d7ebf3644e181aa529744601b52 with delete_type TABLET_DATA_DELETED (replaced by tablet e61b8f296bad46e4bac632f7e784633b) from {username='slave'} at 127.0.0.1:35818
I20261120 03:01:37.929177  3120 tablet_replica.cc:333] T 664b7403286d480788691bddb6816965 P 23741ccd042d4ca49afc3b8d1e26e22c: stopping tablet replica
I20261120 03:01:37.927325  2507 catalog_manager.cc:2218] created HMS catalog entry for table  [id=957365c9d24645e985a308158ec0af66]
I20261120 03:01:37.929520  2985 tablet_service.cc:1542] Processing DeleteTablet for tablet 664b7403286d480788691bddb6816965 with delete_type TABLET_DATA_DELETED (replaced by tablet 1d00a31328c245a88512188ff5f3e584) from {username='slave'} at 127.0.0.1:41990
I20261120 03:01:37.929862  3120 raft_consensus.cc:2440] T 664b7403286d480788691bddb6816965 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:37.930236  2984 tablet_service.cc:1542] Processing DeleteTablet for tablet 1f017d7ebf3644e181aa529744601b52 with delete_type TABLET_DATA_DELETED (replaced by tablet e61b8f296bad46e4bac632f7e784633b) from {username='slave'} at 127.0.0.1:41990
I20261120 03:01:37.930369  3120 raft_consensus.cc:2469] T 664b7403286d480788691bddb6816965 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:37.930907  3119 tablet_replica.cc:333] T 664b7403286d480788691bddb6816965 P c393ca4a78704704bc9b58916a81fc3f: stopping tablet replica
I20261120 03:01:37.931442  3128 raft_consensus.cc:534] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:37.931660  3119 raft_consensus.cc:2440] T 664b7403286d480788691bddb6816965 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:37.932152  3119 raft_consensus.cc:2469] T 664b7403286d480788691bddb6816965 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:37.931893  3128 raft_consensus.cc:556] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.932996  3120 ts_tablet_manager.cc:1962] T 664b7403286d480788691bddb6816965 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:37.933948  2695 tablet_service.cc:1542] Processing DeleteTablet for tablet 664b7403286d480788691bddb6816965 with delete_type TABLET_DATA_DELETED (replaced by tablet 1d00a31328c245a88512188ff5f3e584) from {username='slave'} at 127.0.0.1:38150
I20261120 03:01:37.934657  2694 tablet_service.cc:1542] Processing DeleteTablet for tablet 1f017d7ebf3644e181aa529744601b52 with delete_type TABLET_DATA_DELETED (replaced by tablet e61b8f296bad46e4bac632f7e784633b) from {username='slave'} at 127.0.0.1:38150
I20261120 03:01:37.937152  3119 ts_tablet_manager.cc:1962] T 664b7403286d480788691bddb6816965 P c393ca4a78704704bc9b58916a81fc3f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:37.943534  3118 ts_tablet_manager.cc:1962] T 1f017d7ebf3644e181aa529744601b52 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting tablet data with delete state TABLET_DATA_DELETED
F20261120 03:01:37.946455  2566 hms_notification_log_listener.cc:287] non-monotonous event IDs from HMS: current 1, previous 1; dumping first 2 out of 4 received events: NotificationEvent(eventId=1, eventTime=1795143696, eventType=CREATE_TABLE, dbName=default, tableName=table_1c9622c0bec4498cb886d15294dc5395, message={"server":"","servicePrincipal":"","db":"default","table":"table_1c9622c0bec4498cb886d15294dc5395","tableType":"MANAGED_TABLE","tableObjJson":"{\"1\":{\"str\":\"table_1c9622c0bec4498cb886d15294dc5395\"},\"2\":{\"str\":\"default\"},\"3\":{\"str\":\"slave\"},\"4\":{\"i32\":1795143696},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"key\"},\"2\":{\"str\":\"int\"},\"3\":{\"str\":\"\"}}]},\"2\":{\"str\":\"file:/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/warehouse/table_1c9622c0bec4498cb886d15294dc5395\"},\"3\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"str\":\"\"},\"2\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduSerDe\"},\"3\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"rec\",0]},\"9\":{\"map\":[\"str\",\"str\",7,{\"kudu.table_name\":\"default.table_1c9622c0bec4498cb886d15294dc5395\",\"kudu.table_id\":\"982cb6e783c64717b3882765a530c3eb\",\"transient_lastDdlTime\":\"1795143696\",\"comment\":\"\",\"kudu.master_addresses\":\"127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751\",\"storage_handler\":\"org.apache.hadoop.hive.kudu.KuduStorageHandler\",\"kudu.cluster_id\":\"7f28cd38e32845a18797274c93499aa6\"}]},\"10\":{\"str\":\"\"},\"11\":{\"str\":\"\"},\"12\":{\"str\":\"MANAGED_TABLE\"},\"14\":{\"tf\":0},\"17\":{\"str\":\"hive\"},\"18\":{\"i32\":1}}","timestamp":1795143696,"files":[]}, messageFormat=json-0.2, catName=hive); NotificationEvent(eventId=1, eventTime=1795143696, eventType=CREATE_TABLE, dbName=default, tableName=table_ad2a1b8b11fe4111b57b8eeae744cf96, message={"server":"","servicePrincipal":"","db":"default","table":"table_ad2a1b8b11fe4111b57b8eeae744cf96","tableType":"MANAGED_TABLE","tableObjJson":"{\"1\":{\"str\":\"table_ad2a1b8b11fe4111b57b8eeae744cf96\"},\"2\":{\"str\":\"default\"},\"3\":{\"str\":\"slave\"},\"4\":{\"i32\":1795143696},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"key\"},\"2\":{\"str\":\"int\"},\"3\":{\"str\":\"\"}}]},\"2\":{\"str\":\"file:/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/warehouse/table_ad2a1b8b11fe4111b57b8eeae744cf96\"},\"3\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"str\":\"\"},\"2\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduSerDe\"},\"3\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"rec\",0]},\"9\":{\"map\":[\"str\",\"str\",7,{\"kudu.table_name\":\"default.table_ad2a1b8b11fe4111b57b8eeae744cf96\",\"kudu.table_id\":\"6c71ec67711e43cd8452336d7b135667\",\"transient_lastDdlTime\":\"1795143696\",\"comment\":\"\",\"kudu.master_addresses\":\"127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751\",\"storage_handler\":\"org.apache.hadoop.hive.kudu.KuduStorageHandler\",\"kudu.cluster_id\":\"7f28cd38e32845a18797274c93499aa6\"}]},\"10\":{\"str\":\"\"},\"11\":{\"str\":\"\"},\"12\":{\"str\":\"MANAGED_TABLE\"},\"14\":{\"tf\":0},\"17\":{\"str\":\"hive\"},\"18\":{\"i32\":1}}","timestamp":1795143696,"files":[]}, messageFormat=json-0.2, catName=hive);
*** Check failure stack trace: ***
I20261120 03:01:37.949293  2715 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "7c1a821b612b4325bc34a3a16c284b2d" candidate_uuid: "c393ca4a78704704bc9b58916a81fc3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" is_pre_election: true
I20261120 03:01:37.950132  2715 raft_consensus.cc:2062] T 7c1a821b612b4325bc34a3a16c284b2d P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate c393ca4a78704704bc9b58916a81fc3f in term 0
I20261120 03:01:37.951604  2941 leader_election.cc:304] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c393ca4a78704704bc9b58916a81fc3f, d9ed1d903ef340ac8f1025c2b7df9bdd; no voters: 
I20261120 03:01:37.958652  3128 leader_election.cc:290] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769), 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733)
I20261120 03:01:37.959201  3128 raft_consensus.cc:3024] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:37.959512  3128 raft_consensus.cc:534] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:37.959817  3128 raft_consensus.cc:3296] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:37.960549  3128 raft_consensus.cc:556] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:37.968156  3120 ts_tablet_manager.cc:1975] T 664b7403286d480788691bddb6816965 P 23741ccd042d4ca49afc3b8d1e26e22c: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:37.968585  3120 log.cc:1199] T 664b7403286d480788691bddb6816965 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/664b7403286d480788691bddb6816965
I20261120 03:01:37.969210  3120 ts_tablet_manager.cc:1996] T 664b7403286d480788691bddb6816965 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting consensus metadata
I20261120 03:01:37.980967  2507 rpcz_store.cc:275] Call kudu.master.MasterService.CreateTable from 127.0.0.1:57466 (request call id 16) took 1829 ms. Trace:
I20261120 03:01:37.983268  2495 catalog_manager.cc:5081] TS 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): tablet 664b7403286d480788691bddb6816965 (table default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]) successfully deleted
I20261120 03:01:37.981328  2507 rpcz_store.cc:276] 1120 03:01:36.152033 (+     0us) service_pool.cc:167] Inserting onto call queue
1120 03:01:36.154210 (+  2177us) service_pool.cc:224] Handling call
1120 03:01:36.170641 (+ 16431us) catalog_manager.cc:2145] Acquired catalog manager lock
1120 03:01:36.181649 (+ 11008us) catalog_manager.cc:2191] Created new table and tablet info
1120 03:01:37.927280 (+1745631us) catalog_manager.cc:2217] Created new table in HMS catalog
1120 03:01:37.980253 (+ 52973us) catalog_manager.cc:2251] Wrote table and tablets to system table
1120 03:01:37.980598 (+   345us) catalog_manager.cc:2285] Inserted table and tablets into CatalogManager maps
1120 03:01:37.980902 (+   304us) inbound_call.cc:177] Queueing success response
Related trace 'op':
1120 03:01:37.931107 (+     0us) write_op.cc:183] PREPARE: starting on tablet 00000000000000000000000000000000
1120 03:01:37.931597 (+   490us) write_op.cc:437] Acquiring schema lock in shared mode
1120 03:01:37.931624 (+    27us) write_op.cc:440] Acquired schema lock
1120 03:01:37.931639 (+    15us) tablet.cc:710] Decoding operations
1120 03:01:37.931970 (+   331us) write_op.cc:625] Acquiring the partition lock for write op
1120 03:01:37.932020 (+    50us) write_op.cc:646] Partition lock acquired for write op
1120 03:01:37.932046 (+    26us) tablet.cc:742] Acquiring locks for 2 operations
1120 03:01:37.932281 (+   235us) tablet.cc:758] Row locks acquired
1120 03:01:37.932294 (+    13us) write_op.cc:260] PREPARE: finished
1120 03:01:37.932438 (+   144us) write_op.cc:270] Start()
1120 03:01:37.932528 (+    90us) write_op.cc:276] Timestamp: P: 1795143697932343 usec, L: 0
1120 03:01:37.932544 (+    16us) op_driver.cc:350] REPLICATION: starting
1120 03:01:37.933063 (+   519us) log.cc:844] Serialized 432 byte log entry
1120 03:01:37.977359 (+ 44296us) op_driver.cc:466] REPLICATION: finished
1120 03:01:37.978623 (+  1264us) write_op.cc:301] APPLY: starting
1120 03:01:37.978679 (+    56us) tablet.cc:1451] starting BulkCheckPresence
1120 03:01:37.978786 (+   107us) tablet.cc:1454] finished BulkCheckPresence
1120 03:01:37.978870 (+    84us) tablet.cc:1456] starting ApplyRowOperation cycle
1120 03:01:37.979306 (+   436us) tablet.cc:1467] finished ApplyRowOperation cycle
1120 03:01:37.979434 (+   128us) tablet_metrics.cc:626] ProbeStats: bloom_lookups=0,key_file_lookups=0,delta_file_lookups=0,mrs_lookups=0
1120 03:01:37.979473 (+    39us) write_op.cc:312] APPLY: finished
1120 03:01:37.979796 (+   323us) log.cc:844] Serialized 28 byte log entry
1120 03:01:37.979935 (+   139us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:37.980000 (+    65us) write_op.cc:459] Released schema lock
1120 03:01:37.980030 (+    30us) write_op.cc:346] FINISH: Updating metrics
1120 03:01:37.980256 (+   226us) write_op.cc:494] Releasing partition, row and schema locks
1120 03:01:37.980278 (+    22us) write_op.cc:459] Released schema lock
Metrics: {"Hive Metastore.queue_time_us":1502683,"Hive Metastore.run_cpu_time_us":597,"Hive Metastore.run_wall_time_us":241606,"child_traces":[["op",{"apply.queue_time_us":1112,"apply.run_cpu_time_us":1503,"apply.run_wall_time_us":1609,"num_ops":2,"prepare.queue_time_us":106,"prepare.run_cpu_time_us":2237,"prepare.run_wall_time_us":2237,"raft.queue_time_us":1921,"raft.run_cpu_time_us":15,"raft.run_wall_time_us":15,"replication_time_us":44715}]]}
I20261120 03:01:37.991168  2842 tablet_service.cc:1542] Processing DeleteTablet for tablet 664b7403286d480788691bddb6816965 with delete_type TABLET_DATA_DELETED (replaced by tablet 1d00a31328c245a88512188ff5f3e584) from {username='slave'} at 127.0.0.1:35818
I20261120 03:01:37.992525  3120 tablet_replica.cc:333] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c: stopping tablet replica
I20261120 03:01:37.993191  3120 raft_consensus.cc:2440] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:01:37.993189  2507 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:57466:
table { table_name: "default.Table_a938daa15a8b429190ccb7a61dab9f86" } modify_external_catalogs: true
I20261120 03:01:37.994107  2507 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:57466:
table { table_name: "default.Table_a938daa15a8b429190ccb7a61dab9f86" } modify_external_catalogs: true
I20261120 03:01:37.994277  3120 raft_consensus.cc:2469] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:37.996417  3128 leader_election.cc:290] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 election: Requested vote from peers d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769), 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733)
I20261120 03:01:37.996943  2715 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "7c1a821b612b4325bc34a3a16c284b2d" candidate_uuid: "c393ca4a78704704bc9b58916a81fc3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd"
I20261120 03:01:37.997457  2715 raft_consensus.cc:3296] T 7c1a821b612b4325bc34a3a16c284b2d P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:37.998378  2715 raft_consensus.cc:2062] T 7c1a821b612b4325bc34a3a16c284b2d P d9ed1d903ef340ac8f1025c2b7df9bdd [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate c393ca4a78704704bc9b58916a81fc3f in term 1
I20261120 03:01:38.006412  2941 leader_election.cc:304] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c393ca4a78704704bc9b58916a81fc3f, d9ed1d903ef340ac8f1025c2b7df9bdd; no voters: 
I20261120 03:01:38.007750  3120 ts_tablet_manager.cc:1962] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:38.010416  3160 raft_consensus.cc:3024] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:38.011741  3160 raft_consensus.cc:744] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [term 1 LEADER]: Becoming Leader. State: Replica: c393ca4a78704704bc9b58916a81fc3f, State: Running, Role: LEADER
I20261120 03:01:38.012652  3160 consensus_queue.cc:238] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
W20261120 03:01:38.036146  2502 catalog_manager.cc:5689] Ignoring report from unknown tablet 7c1a821b612b4325bc34a3a16c284b2d
I20261120 03:01:38.039161  3128 consensus_queue.cc:1049] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20261120 03:01:38.040925  3118 ts_tablet_manager.cc:1975] T 1f017d7ebf3644e181aa529744601b52 P d9ed1d903ef340ac8f1025c2b7df9bdd: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:38.041302  3118 log.cc:1199] T 1f017d7ebf3644e181aa529744601b52 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/1f017d7ebf3644e181aa529744601b52
I20261120 03:01:38.041989  3118 ts_tablet_manager.cc:1996] T 1f017d7ebf3644e181aa529744601b52 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting consensus metadata
W20261120 03:01:38.053050  2502 catalog_manager.cc:5689] Ignoring report from unknown tablet 7c1a821b612b4325bc34a3a16c284b2d
W20261120 03:01:38.060135  2939 leader_election.cc:343] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d
W20261120 03:01:38.061244  2939 leader_election.cc:343] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d
I20261120 03:01:38.065635  2497 catalog_manager.cc:5081] TS d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769): tablet 1f017d7ebf3644e181aa529744601b52 (table default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]) successfully deleted
W20261120 03:01:38.065896  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:01:38.066727  3118 tablet_replica.cc:333] T 664b7403286d480788691bddb6816965 P d9ed1d903ef340ac8f1025c2b7df9bdd: stopping tablet replica
I20261120 03:01:38.067441  3118 raft_consensus.cc:2440] T 664b7403286d480788691bddb6816965 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:38.068004  3118 raft_consensus.cc:2469] T 664b7403286d480788691bddb6816965 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:38.070194  3118 ts_tablet_manager.cc:1962] T 664b7403286d480788691bddb6816965 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:38.074225  3120 ts_tablet_manager.cc:1975] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:38.074615  3120 log.cc:1199] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/1f017d7ebf3644e181aa529744601b52
I20261120 03:01:38.081168  3120 ts_tablet_manager.cc:1996] T 1f017d7ebf3644e181aa529744601b52 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting consensus metadata
W20261120 03:01:38.081884  2502 catalog_manager.cc:5689] Ignoring report from unknown tablet 7c1a821b612b4325bc34a3a16c284b2d
I20261120 03:01:38.083818  3119 ts_tablet_manager.cc:1975] T 664b7403286d480788691bddb6816965 P c393ca4a78704704bc9b58916a81fc3f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:38.084167  2495 catalog_manager.cc:5081] TS 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): tablet 1f017d7ebf3644e181aa529744601b52 (table default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]) successfully deleted
I20261120 03:01:38.084304  3119 log.cc:1199] T 664b7403286d480788691bddb6816965 P c393ca4a78704704bc9b58916a81fc3f: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/664b7403286d480788691bddb6816965
I20261120 03:01:38.085050  3119 ts_tablet_manager.cc:1996] T 664b7403286d480788691bddb6816965 P c393ca4a78704704bc9b58916a81fc3f: Deleting consensus metadata
I20261120 03:01:38.087652  2497 catalog_manager.cc:5081] TS c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993): tablet 664b7403286d480788691bddb6816965 (table default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]) successfully deleted
I20261120 03:01:38.094586  2842 tablet_service.cc:1495] Processing CreateTablet for tablet 9cc585adb4df48d18b69d8102382bf2e (DEFAULT_TABLE table=default.table_a938daa15a8b429190ccb7a61dab9f86 [id=957365c9d24645e985a308158ec0af66]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:38.095815  2842 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9cc585adb4df48d18b69d8102382bf2e. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:01:38.095814  2495 catalog_manager.cc:5048] TS 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): delete failed for tablet 664b7403286d480788691bddb6816965 because the tablet was not found. No further retry: Not found: Tablet not found: 664b7403286d480788691bddb6816965
W20261120 03:01:38.101904  2502 catalog_manager.cc:5689] Ignoring report from unknown tablet 7c1a821b612b4325bc34a3a16c284b2d
I20261120 03:01:38.113109  3119 tablet_replica.cc:333] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f: stopping tablet replica
I20261120 03:01:38.113852  3119 raft_consensus.cc:2440] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:38.114418  3119 raft_consensus.cc:2469] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:38.114334  2984 tablet_service.cc:1495] Processing CreateTablet for tablet 9cc585adb4df48d18b69d8102382bf2e (DEFAULT_TABLE table=default.table_a938daa15a8b429190ccb7a61dab9f86 [id=957365c9d24645e985a308158ec0af66]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:38.115571  2984 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9cc585adb4df48d18b69d8102382bf2e. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:38.127801  3119 ts_tablet_manager.cc:1962] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:38.132216  2694 tablet_service.cc:1495] Processing CreateTablet for tablet 9cc585adb4df48d18b69d8102382bf2e (DEFAULT_TABLE table=default.table_a938daa15a8b429190ccb7a61dab9f86 [id=957365c9d24645e985a308158ec0af66]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:38.133751  2694 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9cc585adb4df48d18b69d8102382bf2e. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:38.139705  2695 tablet_service.cc:1495] Processing CreateTablet for tablet 1d00a31328c245a88512188ff5f3e584 (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:38.140786  2695 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1d00a31328c245a88512188ff5f3e584. 1 dirs total, 0 dirs full, 0 dirs failed
    @     0x7f0343131920  google::LogMessage::Flush() at ??:0
I20261120 03:01:38.153407  3119 ts_tablet_manager.cc:1975] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:38.153806  3119 log.cc:1199] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/1f017d7ebf3644e181aa529744601b52
I20261120 03:01:38.154543  3119 ts_tablet_manager.cc:1996] T 1f017d7ebf3644e181aa529744601b52 P c393ca4a78704704bc9b58916a81fc3f: Deleting consensus metadata
I20261120 03:01:38.157455  2497 catalog_manager.cc:5081] TS c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993): tablet 1f017d7ebf3644e181aa529744601b52 (table default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]) successfully deleted
I20261120 03:01:38.162920  2502 master_service.cc:965] ReplaceTablet: received request to replace tablet 7c1a821b612b4325bc34a3a16c284b2d from {username='slave'} at 127.0.0.1:57466
I20261120 03:01:38.170150  2841 tablet_service.cc:1495] Processing CreateTablet for tablet 1d00a31328c245a88512188ff5f3e584 (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:38.171314  2841 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1d00a31328c245a88512188ff5f3e584. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:38.171079  2985 tablet_service.cc:1495] Processing CreateTablet for tablet 1d00a31328c245a88512188ff5f3e584 (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:38.172201  2985 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 1d00a31328c245a88512188ff5f3e584. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:38.175099  2502 master_service.cc:965] ReplaceTablet: received request to replace tablet 7c1a821b612b4325bc34a3a16c284b2d from {username='slave'} at 127.0.0.1:57466
I20261120 03:01:38.186389  3114 tablet_bootstrap.cc:492] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f: Bootstrap starting.
I20261120 03:01:38.193832  2693 tablet_service.cc:1495] Processing CreateTablet for tablet e61b8f296bad46e4bac632f7e784633b (DEFAULT_TABLE table=default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:38.195113  2693 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e61b8f296bad46e4bac632f7e784633b. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:38.195395  3112 tablet_bootstrap.cc:492] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c: Bootstrap starting.
I20261120 03:01:38.199126  2502 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:57466:
name: "default.Table_ef2c937cb5564121ad7b40189665d8ba"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:38.201156  2843 tablet_service.cc:1495] Processing CreateTablet for tablet e61b8f296bad46e4bac632f7e784633b (DEFAULT_TABLE table=default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:38.202310  2843 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e61b8f296bad46e4bac632f7e784633b. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:38.202443  3112 tablet_bootstrap.cc:654] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:38.205974  2983 tablet_service.cc:1495] Processing CreateTablet for tablet e61b8f296bad46e4bac632f7e784633b (DEFAULT_TABLE table=default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:38.207080  2983 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet e61b8f296bad46e4bac632f7e784633b. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:38.209818  3113 tablet_bootstrap.cc:492] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd: Bootstrap starting.
I20261120 03:01:38.222816  3118 ts_tablet_manager.cc:1975] T 664b7403286d480788691bddb6816965 P d9ed1d903ef340ac8f1025c2b7df9bdd: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 0.0
I20261120 03:01:38.223217  3118 log.cc:1199] T 664b7403286d480788691bddb6816965 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/664b7403286d480788691bddb6816965
I20261120 03:01:38.223845  3118 ts_tablet_manager.cc:1996] T 664b7403286d480788691bddb6816965 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting consensus metadata
I20261120 03:01:38.226876  2497 catalog_manager.cc:5081] TS d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769): tablet 664b7403286d480788691bddb6816965 (table default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]) successfully deleted
W20261120 03:01:38.227603  2497 catalog_manager.cc:5048] TS d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769): delete failed for tablet 1f017d7ebf3644e181aa529744601b52 because the tablet was not found. No further retry: Not found: Tablet not found: 1f017d7ebf3644e181aa529744601b52
I20261120 03:01:38.230062  3113 tablet_bootstrap.cc:654] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd: Neither blocks nor log segments found. Creating new log.
    @     0x7f034313168f  google::LogMessage::~LogMessage() at ??:0
I20261120 03:01:38.236135  3114 tablet_bootstrap.cc:654] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:38.241735  3113 tablet_bootstrap.cc:492] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd: No bootstrap required, opened a new log
I20261120 03:01:38.242120  3113 ts_tablet_manager.cc:1449] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent bootstrapping tablet: real 0.032s	user 0.009s	sys 0.005s
I20261120 03:01:38.244333  3113 raft_consensus.cc:366] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.244897  3113 raft_consensus.cc:392] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:38.245131  3113 raft_consensus.cc:787] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9ed1d903ef340ac8f1025c2b7df9bdd, State: Initialized, Role: FOLLOWER
I20261120 03:01:38.245704  3113 consensus_queue.cc:261] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.248040  3113 ts_tablet_manager.cc:1480] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.000s
I20261120 03:01:38.248780  3113 tablet_bootstrap.cc:492] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd: Bootstrap starting.
I20261120 03:01:38.254019  3113 tablet_bootstrap.cc:654] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:38.255102  3114 tablet_bootstrap.cc:492] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f: No bootstrap required, opened a new log
I20261120 03:01:38.255234  3112 tablet_bootstrap.cc:492] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c: No bootstrap required, opened a new log
I20261120 03:01:38.255492  3114 ts_tablet_manager.cc:1449] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f: Time spent bootstrapping tablet: real 0.069s	user 0.009s	sys 0.001s
I20261120 03:01:38.255635  3112 ts_tablet_manager.cc:1449] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent bootstrapping tablet: real 0.060s	user 0.010s	sys 0.027s
I20261120 03:01:38.257839  3114 raft_consensus.cc:366] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.258450  3114 raft_consensus.cc:392] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:38.258687  3114 raft_consensus.cc:787] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c393ca4a78704704bc9b58916a81fc3f, State: Initialized, Role: FOLLOWER
I20261120 03:01:38.258646  3112 raft_consensus.cc:366] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.259302  3112 raft_consensus.cc:392] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:38.259545  3112 raft_consensus.cc:787] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23741ccd042d4ca49afc3b8d1e26e22c, State: Initialized, Role: FOLLOWER
I20261120 03:01:38.260298  3112 consensus_queue.cc:261] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.261291  3114 consensus_queue.cc:261] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.265594  3112 ts_tablet_manager.cc:1480] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent starting tablet: real 0.010s	user 0.007s	sys 0.000s
I20261120 03:01:38.266364  3112 tablet_bootstrap.cc:492] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c: Bootstrap starting.
I20261120 03:01:38.268682  3114 ts_tablet_manager.cc:1480] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f: Time spent starting tablet: real 0.013s	user 0.003s	sys 0.002s
I20261120 03:01:38.269555  3114 tablet_bootstrap.cc:492] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f: Bootstrap starting.
I20261120 03:01:38.275202  3114 tablet_bootstrap.cc:654] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:38.291666  3112 tablet_bootstrap.cc:654] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c: Neither blocks nor log segments found. Creating new log.
W20261120 03:01:38.309186  3122 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
I20261120 03:01:38.314421  3112 tablet_bootstrap.cc:492] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c: No bootstrap required, opened a new log
I20261120 03:01:38.314827  3112 ts_tablet_manager.cc:1449] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent bootstrapping tablet: real 0.049s	user 0.010s	sys 0.000s
I20261120 03:01:38.317101  3112 raft_consensus.cc:366] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.317684  3112 raft_consensus.cc:392] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:38.317907  3112 raft_consensus.cc:787] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23741ccd042d4ca49afc3b8d1e26e22c, State: Initialized, Role: FOLLOWER
I20261120 03:01:38.320962  3112 consensus_queue.cc:261] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
W20261120 03:01:38.309998  3122 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20261120 03:01:38.322525  3122 flags.cc:432] Enabled unsafe flag: --never_fsync=true
I20261120 03:01:38.328559  3112 ts_tablet_manager.cc:1480] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent starting tablet: real 0.013s	user 0.005s	sys 0.000s
I20261120 03:01:38.329396  3112 tablet_bootstrap.cc:492] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c: Bootstrap starting.
    @     0x7f034c8313ac  kudu::master::HmsNotificationLogListenerTask::Poll() at ??:0
I20261120 03:01:38.365012  3112 tablet_bootstrap.cc:654] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:38.368425  3114 tablet_bootstrap.cc:492] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f: No bootstrap required, opened a new log
I20261120 03:01:38.368826  3114 ts_tablet_manager.cc:1449] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f: Time spent bootstrapping tablet: real 0.099s	user 0.035s	sys 0.060s
I20261120 03:01:38.369374  3113 tablet_bootstrap.cc:492] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd: No bootstrap required, opened a new log
I20261120 03:01:38.369753  3113 ts_tablet_manager.cc:1449] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent bootstrapping tablet: real 0.121s	user 0.020s	sys 0.011s
I20261120 03:01:38.372151  2505 master_service.cc:965] ReplaceTablet: received request to replace tablet 7c1a821b612b4325bc34a3a16c284b2d from {username='slave'} at 127.0.0.1:57466
I20261120 03:01:38.372219  3113 raft_consensus.cc:366] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.372828  3113 raft_consensus.cc:392] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:38.373081  3113 raft_consensus.cc:787] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9ed1d903ef340ac8f1025c2b7df9bdd, State: Initialized, Role: FOLLOWER
I20261120 03:01:38.373140  3114 raft_consensus.cc:366] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.373763  3114 raft_consensus.cc:392] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:38.373993  3114 raft_consensus.cc:787] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c393ca4a78704704bc9b58916a81fc3f, State: Initialized, Role: FOLLOWER
I20261120 03:01:38.374612  3114 consensus_queue.cc:261] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.377321  3114 ts_tablet_manager.cc:1480] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f: Time spent starting tablet: real 0.008s	user 0.006s	sys 0.000s
I20261120 03:01:38.381026  3113 consensus_queue.cc:261] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.385872  2501 master_service.cc:965] ReplaceTablet: received request to replace tablet 7c1a821b612b4325bc34a3a16c284b2d from {username='slave'} at 127.0.0.1:57466
I20261120 03:01:38.387439  3114 tablet_bootstrap.cc:492] T e61b8f296bad46e4bac632f7e784633b P c393ca4a78704704bc9b58916a81fc3f: Bootstrap starting.
I20261120 03:01:38.392299  3113 ts_tablet_manager.cc:1480] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent starting tablet: real 0.022s	user 0.001s	sys 0.005s
I20261120 03:01:38.393051  3113 tablet_bootstrap.cc:492] T e61b8f296bad46e4bac632f7e784633b P d9ed1d903ef340ac8f1025c2b7df9bdd: Bootstrap starting.
I20261120 03:01:38.403254  3114 tablet_bootstrap.cc:654] T e61b8f296bad46e4bac632f7e784633b P c393ca4a78704704bc9b58916a81fc3f: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:38.407286  3112 tablet_bootstrap.cc:492] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c: No bootstrap required, opened a new log
I20261120 03:01:38.410553  3112 ts_tablet_manager.cc:1449] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent bootstrapping tablet: real 0.081s	user 0.013s	sys 0.001s
I20261120 03:01:38.414422  3113 tablet_bootstrap.cc:654] T e61b8f296bad46e4bac632f7e784633b P d9ed1d903ef340ac8f1025c2b7df9bdd: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:38.414556  3114 tablet_bootstrap.cc:492] T e61b8f296bad46e4bac632f7e784633b P c393ca4a78704704bc9b58916a81fc3f: No bootstrap required, opened a new log
I20261120 03:01:38.414934  3114 ts_tablet_manager.cc:1449] T e61b8f296bad46e4bac632f7e784633b P c393ca4a78704704bc9b58916a81fc3f: Time spent bootstrapping tablet: real 0.028s	user 0.013s	sys 0.003s
I20261120 03:01:38.417260  3114 raft_consensus.cc:366] T e61b8f296bad46e4bac632f7e784633b P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.417886  3114 raft_consensus.cc:392] T e61b8f296bad46e4bac632f7e784633b P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:38.418160  3114 raft_consensus.cc:787] T e61b8f296bad46e4bac632f7e784633b P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c393ca4a78704704bc9b58916a81fc3f, State: Initialized, Role: FOLLOWER
I20261120 03:01:38.418216  3112 raft_consensus.cc:366] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.418931  3112 raft_consensus.cc:392] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:38.419198  3112 raft_consensus.cc:787] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23741ccd042d4ca49afc3b8d1e26e22c, State: Initialized, Role: FOLLOWER
I20261120 03:01:38.418944  3114 consensus_queue.cc:261] T e61b8f296bad46e4bac632f7e784633b P c393ca4a78704704bc9b58916a81fc3f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.419893  3112 consensus_queue.cc:261] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
    @     0x7f034c830b6c  kudu::master::HmsNotificationLogListenerTask::RunLoop() at ??:0
I20261120 03:01:38.428735  3114 ts_tablet_manager.cc:1480] T e61b8f296bad46e4bac632f7e784633b P c393ca4a78704704bc9b58916a81fc3f: Time spent starting tablet: real 0.014s	user 0.005s	sys 0.000s
I20261120 03:01:38.429507  3112 ts_tablet_manager.cc:1480] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent starting tablet: real 0.019s	user 0.005s	sys 0.000s
I20261120 03:01:38.439950  3113 tablet_bootstrap.cc:492] T e61b8f296bad46e4bac632f7e784633b P d9ed1d903ef340ac8f1025c2b7df9bdd: No bootstrap required, opened a new log
W20261120 03:01:38.440100  2496 consensus_peers.cc:597] T 00000000000000000000000000000000 P 86ed298871604dbe84645c0199254db8 -> Peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Couldn't send request to peer 6e18f436ef9945d78f543c3471320ee7. Status: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20261120 03:01:38.440364  3113 ts_tablet_manager.cc:1449] T e61b8f296bad46e4bac632f7e784633b P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent bootstrapping tablet: real 0.047s	user 0.005s	sys 0.003s
I20261120 03:01:38.442761  3113 raft_consensus.cc:366] T e61b8f296bad46e4bac632f7e784633b P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.443361  3113 raft_consensus.cc:392] T e61b8f296bad46e4bac632f7e784633b P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:38.443588  3113 raft_consensus.cc:787] T e61b8f296bad46e4bac632f7e784633b P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9ed1d903ef340ac8f1025c2b7df9bdd, State: Initialized, Role: FOLLOWER
I20261120 03:01:38.444540  3113 consensus_queue.cc:261] T e61b8f296bad46e4bac632f7e784633b P d9ed1d903ef340ac8f1025c2b7df9bdd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.446668  3113 ts_tablet_manager.cc:1480] T e61b8f296bad46e4bac632f7e784633b P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent starting tablet: real 0.006s	user 0.007s	sys 0.000s
    @     0x7f034c834be2  kudu::master::HmsNotificationLogListenerTask::Init()::$_0::operator()() at ??:0
    @     0x7f034c834b9a  _ZNSt3__18__invokeIRZN4kudu6master30HmsNotificationLogListenerTask4InitEvE3$_0JEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOS6_DpOS7_ at ??:0
W20261120 03:01:38.484879  3122 flags.cc:432] Enabled experimental flag: --enable_metadata_cleanup_for_deleted_tables_and_tablets=true
W20261120 03:01:38.485208  3122 flags.cc:432] Enabled experimental flag: --metadata_for_deleted_table_and_tablet_reserved_secs=0
W20261120 03:01:38.485559  3122 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
W20261120 03:01:38.485883  3122 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
W20261120 03:01:38.486241  3122 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
W20261120 03:01:38.486609  3122 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
    @     0x7f034c834b2a  std::__1::__invoke_void_return_wrapper<>::__call<>() at ??:0
    @     0x7f034c834af2  std::__1::__function::__alloc_func<>::operator()() at ??:0
    @     0x7f034c833dee  std::__1::__function::__func<>::operator()() at ??:0
    @           0x327655  std::__1::__function::__value_func<>::operator()() at /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/../sanitizer_common/sanitizer_vector.h:111
    @           0x32747a  std::__1::function<>::operator()() at /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp:294
I20261120 03:01:38.532322  3164 raft_consensus.cc:534] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:38.532823  3164 raft_consensus.cc:556] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.534981  3164 leader_election.cc:290] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769), 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733)
I20261120 03:01:38.536036  2715 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1d00a31328c245a88512188ff5f3e584" candidate_uuid: "c393ca4a78704704bc9b58916a81fc3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" is_pre_election: true
I20261120 03:01:38.536314  3122 master_runner.cc:387] Master server non-default flags:
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/logs
--logbuflevel=-1
--logtostderr=true
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.96.84:36533
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/data
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/wal
--hive_metastore_uris=thrift://127.0.0.1:45151
--catalog_manager_bg_task_wait_ms=10
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.126:46565
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/data/info.pb
--webserver_interface=127.0.96.126
--webserver_port=46381
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false

Master server version:
kudu 1.19.0-SNAPSHOT
revision 36b6772d828145f8fbba3c49746c86838683f9ba
build type FASTDEBUG
built by None at 20 Nov 2026 02:45:53 UTC on bdcb31816ec0
build id 14870
TSAN enabled
I20261120 03:01:38.536768  2715 raft_consensus.cc:2062] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate c393ca4a78704704bc9b58916a81fc3f in term 0
I20261120 03:01:38.538257  3122 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20261120 03:01:38.538560  2861 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1d00a31328c245a88512188ff5f3e584" candidate_uuid: "c393ca4a78704704bc9b58916a81fc3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" is_pre_election: true
I20261120 03:01:38.538656  2941 leader_election.cc:304] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c393ca4a78704704bc9b58916a81fc3f, d9ed1d903ef340ac8f1025c2b7df9bdd; no voters: 
I20261120 03:01:38.539388  2861 raft_consensus.cc:2062] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate c393ca4a78704704bc9b58916a81fc3f in term 0
I20261120 03:01:38.540655  3122 file_cache.cc:504] Constructed file cache file cache with capacity 419430
I20261120 03:01:38.541005  3164 raft_consensus.cc:3024] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:38.541345  3164 raft_consensus.cc:534] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:38.541640  3164 raft_consensus.cc:3296] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Advancing to term 1
    @     0x7f03438437db  kudu::Thread::SuperviseThread() at ??:0
    @           0x277de0  __tsan_thread_start_func at ??:0
I20261120 03:01:38.551017  3164 raft_consensus.cc:556] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.554803  2715 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1d00a31328c245a88512188ff5f3e584" candidate_uuid: "c393ca4a78704704bc9b58916a81fc3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd"
I20261120 03:01:38.555383  2715 raft_consensus.cc:3296] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:38.556262  2715 raft_consensus.cc:2062] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate c393ca4a78704704bc9b58916a81fc3f in term 1
W20261120 03:01:38.558001  3187 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:01:38.563067  3164 leader_election.cc:290] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 election: Requested vote from peers d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769), 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733)
I20261120 03:01:38.564371  2861 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "1d00a31328c245a88512188ff5f3e584" candidate_uuid: "c393ca4a78704704bc9b58916a81fc3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "23741ccd042d4ca49afc3b8d1e26e22c"
    @     0x7f0344fb16db  start_thread at ??:0
I20261120 03:01:38.564936  2861 raft_consensus.cc:3296] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:38.570668  2861 raft_consensus.cc:2062] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate c393ca4a78704704bc9b58916a81fc3f in term 1
I20261120 03:01:38.574707  3123 raft_consensus.cc:534] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:38.575173  3123 raft_consensus.cc:556] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.578037  2714 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "e61b8f296bad46e4bac632f7e784633b" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" is_pre_election: true
I20261120 03:01:38.578595  2939 leader_election.cc:304] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 23741ccd042d4ca49afc3b8d1e26e22c, c393ca4a78704704bc9b58916a81fc3f; no voters: 
I20261120 03:01:38.578812  2714 raft_consensus.cc:2062] T e61b8f296bad46e4bac632f7e784633b P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 23741ccd042d4ca49afc3b8d1e26e22c in term 0
I20261120 03:01:38.579481  3164 raft_consensus.cc:3024] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:38.579924  3164 raft_consensus.cc:744] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [term 1 LEADER]: Becoming Leader. State: Replica: c393ca4a78704704bc9b58916a81fc3f, State: Running, Role: LEADER
    @     0x7f03402e761f  clone at ??:0
I20261120 03:01:38.580696  3164 consensus_queue.cc:238] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.583195  2799 leader_election.cc:304] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 23741ccd042d4ca49afc3b8d1e26e22c, d9ed1d903ef340ac8f1025c2b7df9bdd; no voters: 
I20261120 03:01:38.585359  3123 leader_election.cc:290] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769), c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993)
I20261120 03:01:38.585855  3123 raft_consensus.cc:3024] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:38.586185  3123 raft_consensus.cc:534] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:38.586483  3123 raft_consensus.cc:3296] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:38.587194  3123 raft_consensus.cc:556] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.588814  3004 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "e61b8f296bad46e4bac632f7e784633b" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c393ca4a78704704bc9b58916a81fc3f" is_pre_election: true
I20261120 03:01:38.589524  3004 raft_consensus.cc:2062] T e61b8f296bad46e4bac632f7e784633b P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 23741ccd042d4ca49afc3b8d1e26e22c in term 0
I20261120 03:01:38.625967  2715 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "e61b8f296bad46e4bac632f7e784633b" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd"
I20261120 03:01:38.626590  2715 raft_consensus.cc:3296] T e61b8f296bad46e4bac632f7e784633b P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:38.627506  2715 raft_consensus.cc:2062] T e61b8f296bad46e4bac632f7e784633b P d9ed1d903ef340ac8f1025c2b7df9bdd [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 23741ccd042d4ca49afc3b8d1e26e22c in term 1
I20261120 03:01:38.635447  3004 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "e61b8f296bad46e4bac632f7e784633b" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c393ca4a78704704bc9b58916a81fc3f"
I20261120 03:01:38.635990  3004 raft_consensus.cc:3296] T e61b8f296bad46e4bac632f7e784633b P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:38.637035  3004 raft_consensus.cc:2062] T e61b8f296bad46e4bac632f7e784633b P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 23741ccd042d4ca49afc3b8d1e26e22c in term 1
I20261120 03:01:38.640115  3128 consensus_queue.cc:1049] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [LEADER]: Connected to new peer: Peer: permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:38.652397  3128 consensus_queue.cc:1049] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:38.617843  3123 leader_election.cc:290] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 election: Requested vote from peers d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769), c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993)
I20261120 03:01:38.688490  2799 leader_election.cc:304] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 23741ccd042d4ca49afc3b8d1e26e22c, c393ca4a78704704bc9b58916a81fc3f; no voters: 
I20261120 03:01:38.689548  3123 raft_consensus.cc:3024] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:38.690022  3123 raft_consensus.cc:744] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 LEADER]: Becoming Leader. State: Replica: 23741ccd042d4ca49afc3b8d1e26e22c, State: Running, Role: LEADER
I20261120 03:01:38.690974  3123 consensus_queue.cc:238] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.719009  3192 consensus_queue.cc:1049] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [LEADER]: Connected to new peer: Peer: permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:38.742508  3164 raft_consensus.cc:534] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:38.743064  3164 raft_consensus.cc:556] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.746452  2862 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9cc585adb4df48d18b69d8102382bf2e" candidate_uuid: "c393ca4a78704704bc9b58916a81fc3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" is_pre_election: true
I20261120 03:01:38.747299  2862 raft_consensus.cc:2062] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate c393ca4a78704704bc9b58916a81fc3f in term 0
I20261120 03:01:38.748662  2939 leader_election.cc:304] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 23741ccd042d4ca49afc3b8d1e26e22c, c393ca4a78704704bc9b58916a81fc3f; no voters: 
I20261120 03:01:38.749775  3128 raft_consensus.cc:3024] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:38.750222  3128 raft_consensus.cc:534] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:38.750569  3128 raft_consensus.cc:3296] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:38.789861  3128 raft_consensus.cc:556] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.794759  3164 leader_election.cc:290] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733), d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769)
I20261120 03:01:38.794912  2715 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9cc585adb4df48d18b69d8102382bf2e" candidate_uuid: "c393ca4a78704704bc9b58916a81fc3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" is_pre_election: true
I20261120 03:01:38.795614  2715 raft_consensus.cc:2062] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate c393ca4a78704704bc9b58916a81fc3f in term 0
I20261120 03:01:38.798338  3192 consensus_queue.cc:1049] T e61b8f296bad46e4bac632f7e784633b P 23741ccd042d4ca49afc3b8d1e26e22c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:38.831465  3203 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 86ed298871604dbe84645c0199254db8)
I20261120 03:01:38.831910  3203 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
W20261120 03:01:38.883287  2418 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
I20261120 03:01:38.933514  3203 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
I20261120 03:01:38.941758  3128 leader_election.cc:290] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 election: Requested vote from peers 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733), d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769)
I20261120 03:01:38.944602  2861 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9cc585adb4df48d18b69d8102382bf2e" candidate_uuid: "c393ca4a78704704bc9b58916a81fc3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "23741ccd042d4ca49afc3b8d1e26e22c"
I20261120 03:01:38.945264  2861 raft_consensus.cc:3296] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:38.953347  2861 raft_consensus.cc:2062] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate c393ca4a78704704bc9b58916a81fc3f in term 1
I20261120 03:01:38.955433  2939 leader_election.cc:304] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 23741ccd042d4ca49afc3b8d1e26e22c, c393ca4a78704704bc9b58916a81fc3f; no voters: 
I20261120 03:01:38.956492  3128 raft_consensus.cc:3024] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:38.956990  3128 raft_consensus.cc:744] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [term 1 LEADER]: Becoming Leader. State: Replica: c393ca4a78704704bc9b58916a81fc3f, State: Running, Role: LEADER
I20261120 03:01:38.957861  3128 consensus_queue.cc:238] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:38.961233  2714 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9cc585adb4df48d18b69d8102382bf2e" candidate_uuid: "c393ca4a78704704bc9b58916a81fc3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd"
I20261120 03:01:38.962160  2714 raft_consensus.cc:3296] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:38.963344  2714 raft_consensus.cc:2062] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate c393ca4a78704704bc9b58916a81fc3f in term 1
I20261120 03:01:38.979377  3212 consensus_queue.cc:1049] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:38.982399  3164 consensus_queue.cc:1049] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [LEADER]: Connected to new peer: Peer: permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20261120 03:01:39.120123  3060 connection.cc:573] client connection to 127.0.96.124:33751 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:39.124837  3060 master_proxy_rpc.cc:203] Re-attempting ReplaceTablet request to leader Master (127.0.96.124:33751)
W20261120 03:01:39.125762  3060 master_proxy_rpc.cc:203] Re-attempting CreateTable request to leader Master (127.0.96.124:33751)
W20261120 03:01:39.127009  3060 master_proxy_rpc.cc:203] Re-attempting DeleteTable request to leader Master (127.0.96.124:33751)
W20261120 03:01:39.132058  3060 master_proxy_rpc.cc:203] Re-attempting AlterTable request to leader Master (127.0.96.124:33751)
W20261120 03:01:39.139863  2938 connection.cc:573] client connection to 127.0.96.124:33751 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:39.143740  2795 connection.cc:573] client connection to 127.0.96.124:33751 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:39.148495  2416 connection.cc:573] client connection to 127.0.96.124:33751 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:39.148504  2648 connection.cc:573] client connection to 127.0.96.124:33751 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:39.154712  3121 negotiation.cc:336] Failed RPC negotiation. Trace:
1120 03:01:39.141889 (+     0us) reactor.cc:730] Submitting negotiation task for client connection to 127.0.96.124:33751 (local address 127.0.96.67:37061)
1120 03:01:39.142922 (+  1033us) negotiation.cc:107] Waiting for socket to connect
1120 03:01:39.142970 (+    48us) client_negotiation.cc:175] Beginning negotiation
1120 03:01:39.143231 (+   261us) client_negotiation.cc:262] Sending NEGOTIATE NegotiatePB request
1120 03:01:39.154581 (+ 11350us) negotiation.cc:326] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: BlockingRecv error: recv error from unknown peer: Connection reset by peer (error 104)
Metrics: {"client-negotiator.queue_time_us":407}
W20261120 03:01:39.154716  3223 negotiation.cc:336] Failed RPC negotiation. Trace:
1120 03:01:39.138644 (+     0us) reactor.cc:730] Submitting negotiation task for client connection to 127.0.96.124:33751 (local address 127.0.0.1:57470)
1120 03:01:39.142013 (+  3369us) negotiation.cc:107] Waiting for socket to connect
1120 03:01:39.142059 (+    46us) client_negotiation.cc:175] Beginning negotiation
1120 03:01:39.142500 (+   441us) client_negotiation.cc:262] Sending NEGOTIATE NegotiatePB request
1120 03:01:39.154515 (+ 12015us) negotiation.cc:326] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: BlockingRecv error: recv error from unknown peer: Connection reset by peer (error 104)
Metrics: {"client-negotiator.queue_time_us":2783,"threads_started":1}
W20261120 03:01:39.157720  3139 negotiation.cc:336] Failed RPC negotiation. Trace:
1120 03:01:39.145825 (+     0us) reactor.cc:730] Submitting negotiation task for client connection to 127.0.96.124:33751 (local address 127.0.96.66:55477)
1120 03:01:39.146819 (+   994us) negotiation.cc:107] Waiting for socket to connect
1120 03:01:39.146864 (+    45us) client_negotiation.cc:175] Beginning negotiation
1120 03:01:39.147107 (+   243us) client_negotiation.cc:262] Sending NEGOTIATE NegotiatePB request
1120 03:01:39.157499 (+ 10392us) negotiation.cc:326] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: BlockingRecv error: recv error from unknown peer: Connection reset by peer (error 104)
Metrics: {"client-negotiator.queue_time_us":382}
W20261120 03:01:39.161338  3211 negotiation.cc:336] Failed RPC negotiation. Trace:
1120 03:01:39.151176 (+     0us) reactor.cc:730] Submitting negotiation task for client connection to 127.0.96.124:33751 (local address 127.0.0.1:57472)
1120 03:01:39.152317 (+  1141us) negotiation.cc:107] Waiting for socket to connect
1120 03:01:39.152360 (+    43us) client_negotiation.cc:175] Beginning negotiation
1120 03:01:39.152695 (+   335us) client_negotiation.cc:262] Sending NEGOTIATE NegotiatePB request
1120 03:01:39.161169 (+  8474us) negotiation.cc:326] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: BlockingRecv error: recv error from unknown peer: Connection reset by peer (error 104)
Metrics: {"client-negotiator.queue_time_us":563}
W20261120 03:01:39.163055  2416 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: BlockingRecv error: recv error from unknown peer: Connection reset by peer (error 104)
I20261120 03:01:39.163461  2416 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:01:39.164259  3203 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20261120 03:01:39.165259  3124 negotiation.cc:336] Failed RPC negotiation. Trace:
1120 03:01:39.151361 (+     0us) reactor.cc:730] Submitting negotiation task for client connection to 127.0.96.124:33751 (local address 127.0.96.65:47503)
1120 03:01:39.152162 (+   801us) negotiation.cc:107] Waiting for socket to connect
1120 03:01:39.152215 (+    53us) client_negotiation.cc:175] Beginning negotiation
1120 03:01:39.152555 (+   340us) client_negotiation.cc:262] Sending NEGOTIATE NegotiatePB request
1120 03:01:39.165108 (+ 12553us) negotiation.cc:326] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: BlockingRecv error: recv error from unknown peer: Connection reset by peer (error 104)
Metrics: {"client-negotiator.queue_time_us":168}
I20261120 03:01:39.777483  3226 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 86ed298871604dbe84645c0199254db8)
I20261120 03:01:39.797346  3226 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:39.826402  3226 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:01:39.841598  2418 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:01:39.842538  2939 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.0.96.66:43733, user_credentials={real_user=slave}} blocked reactor thread for 60771.1us
W20261120 03:01:39.843427  2416 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:39.843822  2416 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:01:39.846127  3226 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20261120 03:01:38.560880  3188 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20261120 03:01:39.959357  3185 debug-util.cc:398] Leaking SignalData structure 0x7b0800032ec0 after lost signal to thread 3122
W20261120 03:01:40.571925  3185 kernel_stack_watchdog.cc:198] Thread 3122 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 395ms:
Kernel stack:
(could not read kernel stack)
User stack:
<Timed out: thread did not respond: maybe it is blocking signals>
W20261120 03:01:40.583775  3122 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 2.022s	user 0.002s	sys 0.000s
W20261120 03:01:40.584184  3122 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 2.022s	user 0.002s	sys 0.000s
W20261120 03:01:40.584199  3189 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2022 milliseconds
W20261120 03:01:40.586237  3191 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:01:40.587114  3122 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
I20261120 03:01:40.588560  3122 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20261120 03:01:40.591260  3122 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20261120 03:01:40.592617  3122 hybrid_clock.cc:648] HybridClock initialized: now 1795143700592561 us; error 39 us; skew 500 ppm
I20261120 03:01:40.593456  3122 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20261120 03:01:40.599514  3122 webserver.cc:577] Webserver started at http://127.0.96.126:46381/ using document root <none> and password file <none>
I20261120 03:01:40.600452  3122 fs_manager.cc:362] Metadata directory not provided
I20261120 03:01:40.600687  3122 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20261120 03:01:40.608587  3122 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.000s
I20261120 03:01:40.613448  3237 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:01:40.614627  3122 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.002s
I20261120 03:01:40.614943  3122 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/data,/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/wal
uuid: "6e18f436ef9945d78f543c3471320ee7"
format_stamp: "Formatted at 2026-11-20 03:01:23 on dist-test-slave-jpwh"
I20261120 03:01:40.616964  3122 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/wal
metadata directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/wal
1 data directories: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20261120 03:01:40.674045  3122 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20261120 03:01:40.675734  3122 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20261120 03:01:40.676158  3122 kserver.cc:163] Server-wide thread pool size limit: 3276
I20261120 03:01:40.783298  3122 rpc_server.cc:307] RPC server started. Bound to: 127.0.96.126:46565
I20261120 03:01:40.783363  3292 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 86ed298871604dbe84645c0199254db8)
I20261120 03:01:40.783423  3291 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.96.126:46565 every 8 connection(s)
I20261120 03:01:40.783846  3292 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:40.786522  3122 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-0/data/info.pb
I20261120 03:01:40.791019   385 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu as pid 3122
I20261120 03:01:40.792428  3292 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
I20261120 03:01:40.808506  3296 sys_catalog.cc:263] Verifying existing consensus state
I20261120 03:01:40.818897  3296 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: Bootstrap starting.
W20261120 03:01:40.837605  2416 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:40.842489  3296 log.cc:826] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: Log is configured to *not* fsync() on all Append() calls
I20261120 03:01:40.861583  3296 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20261120 03:01:40.862651  3296 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: Bootstrap complete.
I20261120 03:01:40.887789  3296 raft_consensus.cc:366] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:40.890937  3296 raft_consensus.cc:787] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6e18f436ef9945d78f543c3471320ee7, State: Initialized, Role: FOLLOWER
I20261120 03:01:40.891898  3296 consensus_queue.cc:261] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:40.894424  3302 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:40.895272  3302 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:01:40.896499  3296 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: configured and running, proceeding with master startup.
I20261120 03:01:40.897876  3267 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a23f052846654c9daf3fb0d90fbb6f09" candidate_term: 3 candidate_status { last_received { term: 2 index: 24 } } ignore_live_leader: false dest_uuid: "6e18f436ef9945d78f543c3471320ee7" is_pre_election: true
I20261120 03:01:40.899116  3267 raft_consensus.cc:2062] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 1 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate a23f052846654c9daf3fb0d90fbb6f09 in term 1
I20261120 03:01:40.900333  2418 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6e18f436ef9945d78f543c3471320ee7, a23f052846654c9daf3fb0d90fbb6f09; no voters: 86ed298871604dbe84645c0199254db8
I20261120 03:01:40.901005  3292 raft_consensus.cc:3024] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 2 FOLLOWER]: Leader pre-election won for term 3
I20261120 03:01:40.901338  3292 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 2 FOLLOWER]: Starting leader election (detected failure of leader 86ed298871604dbe84645c0199254db8)
I20261120 03:01:40.901661  3292 raft_consensus.cc:3296] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 2 FOLLOWER]: Advancing to term 3
I20261120 03:01:40.903296  3292 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:40.911893  3292 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 3 election: Requested vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
I20261120 03:01:40.912730  3267 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a23f052846654c9daf3fb0d90fbb6f09" candidate_term: 3 candidate_status { last_received { term: 2 index: 24 } } ignore_live_leader: false dest_uuid: "6e18f436ef9945d78f543c3471320ee7"
I20261120 03:01:40.913300  3267 raft_consensus.cc:3296] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 1 FOLLOWER]: Advancing to term 3
W20261120 03:01:40.914093  2416 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111) [suppressed 5 similar messages]
I20261120 03:01:40.914665  3267 raft_consensus.cc:2062] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Leader election vote request: granting yes vote for candidate a23f052846654c9daf3fb0d90fbb6f09 in term 3
W20261120 03:01:40.917204  2416 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:40.922369  2418 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6e18f436ef9945d78f543c3471320ee7, a23f052846654c9daf3fb0d90fbb6f09; no voters: 86ed298871604dbe84645c0199254db8
I20261120 03:01:40.923095  3292 raft_consensus.cc:3024] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 3 FOLLOWER]: Leader election won for term 3
I20261120 03:01:40.923596  3292 raft_consensus.cc:744] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 3 LEADER]: Becoming Leader. State: Replica: a23f052846654c9daf3fb0d90fbb6f09, State: Running, Role: LEADER
I20261120 03:01:40.924419  3292 consensus_queue.cc:238] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 24, Committed index: 24, Last appended: 2.24, Last appended by leader: 24, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:40.928578  3305 sys_catalog.cc:455] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a23f052846654c9daf3fb0d90fbb6f09. Latest consensus state: current_term: 3 leader_uuid: "a23f052846654c9daf3fb0d90fbb6f09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:40.929344  3305 sys_catalog.cc:458] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: This master's current role is: LEADER
I20261120 03:01:40.934846  3321 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
I20261120 03:01:40.940735  3267 raft_consensus.cc:1338] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Refusing update from remote peer a23f052846654c9daf3fb0d90fbb6f09: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 3 index: 25. (index mismatch)
I20261120 03:01:40.943985  3292 consensus_queue.cc:1049] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 25, Last known committed idx: 4, Time since last communication: 0.001s
W20261120 03:01:40.945155  2416 consensus_peers.cc:597] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 -> Peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Couldn't send request to peer 86ed298871604dbe84645c0199254db8. Status: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20261120 03:01:40.953428  3320 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
I20261120 03:01:40.963382  3320 catalog_manager.cc:694] Loaded metadata for table default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]
I20261120 03:01:40.965133  3320 catalog_manager.cc:694] Loaded metadata for table default.table_a938daa15a8b429190ccb7a61dab9f86 [id=957365c9d24645e985a308158ec0af66]
I20261120 03:01:40.966778  3320 catalog_manager.cc:694] Loaded metadata for table default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]
I20261120 03:01:40.972463  3320 tablet_loader.cc:96] loaded metadata for tablet 1d00a31328c245a88512188ff5f3e584 (table default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3])
I20261120 03:01:40.973482  3320 tablet_loader.cc:96] loaded metadata for tablet 9cc585adb4df48d18b69d8102382bf2e (table default.table_a938daa15a8b429190ccb7a61dab9f86 [id=957365c9d24645e985a308158ec0af66])
I20261120 03:01:40.974679  3320 tablet_loader.cc:96] loaded metadata for tablet e61b8f296bad46e4bac632f7e784633b (table default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb])
I20261120 03:01:40.975670  3320 catalog_manager.cc:1535] Initializing Kudu cluster ID...
I20261120 03:01:40.978888  3320 catalog_manager.cc:1310] Loaded cluster ID: 7f28cd38e32845a18797274c93499aa6
I20261120 03:01:40.979115  3320 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
I20261120 03:01:40.982834  3320 catalog_manager.cc:1555] Loading token signing keys...
I20261120 03:01:40.987476  3320 catalog_manager.cc:6248] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09: Loaded TSK: 0
I20261120 03:01:40.988829  3320 catalog_manager.cc:1565] Initializing in-progress tserver states...
I20261120 03:01:40.993762  3320 catalog_manager.cc:1578] Loading latest processed Hive Metastore notification log event ID...
I20261120 03:01:40.994256  3302 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a23f052846654c9daf3fb0d90fbb6f09. Latest consensus state: current_term: 3 leader_uuid: "a23f052846654c9daf3fb0d90fbb6f09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:40.994969  3302 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:01:40.998448  3320 catalog_manager.cc:6086] Last processed Hive Metastore notification event ID: -1
I20261120 03:01:41.000049  3325 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "a23f052846654c9daf3fb0d90fbb6f09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:41.001066  3325 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:01:41.002524  3302 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "a23f052846654c9daf3fb0d90fbb6f09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:41.006203  3302 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:01:41.014623  3321 catalog_manager.cc:1310] Loaded cluster ID: 7f28cd38e32845a18797274c93499aa6
I20261120 03:01:41.014907  3321 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: loading cluster ID for follower catalog manager: success
I20261120 03:01:41.034940  3292 sys_catalog.cc:455] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "a23f052846654c9daf3fb0d90fbb6f09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:41.035676  3292 sys_catalog.cc:458] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: This master's current role is: LEADER
I20261120 03:01:41.038946  3321 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: acquiring CA information for follower catalog manager: success
I20261120 03:01:41.042557  2432 master_service.cc:965] ReplaceTablet: received request to replace tablet 1d00a31328c245a88512188ff5f3e584 from {username='slave'} at 127.0.0.1:59970
I20261120 03:01:41.045883  3321 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
I20261120 03:01:41.057183  2910 heartbeater.cc:345] Connected to a master server at 127.0.96.126:46565
I20261120 03:01:41.064713  2425 master_service.cc:965] ReplaceTablet: received request to replace tablet 1d00a31328c245a88512188ff5f3e584 from {username='slave'} at 127.0.0.1:59970
I20261120 03:01:41.065135  2766 heartbeater.cc:345] Connected to a master server at 127.0.96.126:46565
I20261120 03:01:41.087525  3256 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" instance_seqno: 1795143691680008) as {username='slave'} at 127.0.96.65:44699; Asking this server to re-register.
I20261120 03:01:41.089311  3255 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" instance_seqno: 1795143693763750) as {username='slave'} at 127.0.96.66:36315; Asking this server to re-register.
I20261120 03:01:41.089653  2766 heartbeater.cc:462] Registering TS with master...
I20261120 03:01:41.091200  2910 heartbeater.cc:462] Registering TS with master...
I20261120 03:01:41.094216  3256 ts_manager.cc:210] Registered new tserver with Master: d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769)
I20261120 03:01:41.095300  3254 ts_manager.cc:210] Registered new tserver with Master: 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733)
I20261120 03:01:41.053958  3122 master_runner.cc:187] Error getting master registration for 127.0.96.124:33751: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111), 
I20261120 03:01:41.097800  3122 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:01:41.050351  2430 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:59970:
name: "default.Table_ef2c937cb5564121ad7b40189665d8ba"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:41.051399  2431 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:59970:
name: "default.Table_35bb29085419456eb14b37a3b5c7e92a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:41.053562  2429 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:59970:
table { table_name: "default.Table_a938daa15a8b429190ccb7a61dab9f86" } modify_external_catalogs: true
I20261120 03:01:41.104791  2429 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:59970:
table { table_name: "default.Table_a938daa15a8b429190ccb7a61dab9f86" } modify_external_catalogs: true
I20261120 03:01:41.058734  2428 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:59970:
name: "default.Table_8feab37db9ac453aaa1afb4721505818"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:41.059604  2427 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:59970:
name: "default.Table_4eedec7d9c82426698cb181fa17e7adb"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:41.060478  2426 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:59970:
table { table_name: "default.Table_e387307c289c48c392b4a49829f76842" } new_table_name: "default.Table_11fd9a471c414923bd22c7180c831f74" modify_external_catalogs: true
I20261120 03:01:41.065452  2424 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:59970:
table { table_name: "default.Table_1c9622c0bec4498cb886d15294dc5395" } new_table_name: "default.Table_b9ef32f90b37407999140f1090dc8049" modify_external_catalogs: true
F20261120 03:01:41.116142  2619 hms_notification_log_listener.cc:287] non-monotonous event IDs from HMS: current 1, previous 1; dumping first 2 out of 4 received events: NotificationEvent(eventId=1, eventTime=1795143696, eventType=CREATE_TABLE, dbName=default, tableName=table_1c9622c0bec4498cb886d15294dc5395, message={"server":"","servicePrincipal":"","db":"default","table":"table_1c9622c0bec4498cb886d15294dc5395","tableType":"MANAGED_TABLE","tableObjJson":"{\"1\":{\"str\":\"table_1c9622c0bec4498cb886d15294dc5395\"},\"2\":{\"str\":\"default\"},\"3\":{\"str\":\"slave\"},\"4\":{\"i32\":1795143696},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"key\"},\"2\":{\"str\":\"int\"},\"3\":{\"str\":\"\"}}]},\"2\":{\"str\":\"file:/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/warehouse/table_1c9622c0bec4498cb886d15294dc5395\"},\"3\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"str\":\"\"},\"2\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduSerDe\"},\"3\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"rec\",0]},\"9\":{\"map\":[\"str\",\"str\",7,{\"kudu.table_name\":\"default.table_1c9622c0bec4498cb886d15294dc5395\",\"kudu.table_id\":\"982cb6e783c64717b3882765a530c3eb\",\"transient_lastDdlTime\":\"1795143696\",\"comment\":\"\",\"kudu.master_addresses\":\"127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751\",\"storage_handler\":\"org.apache.hadoop.hive.kudu.KuduStorageHandler\",\"kudu.cluster_id\":\"7f28cd38e32845a18797274c93499aa6\"}]},\"10\":{\"str\":\"\"},\"11\":{\"str\":\"\"},\"12\":{\"str\":\"MANAGED_TABLE\"},\"14\":{\"tf\":0},\"17\":{\"str\":\"hive\"},\"18\":{\"i32\":1}}","timestamp":1795143696,"files":[]}, messageFormat=json-0.2, catName=hive); NotificationEvent(eventId=1, eventTime=1795143696, eventType=CREATE_TABLE, dbName=default, tableName=table_ad2a1b8b11fe4111b57b8eeae744cf96, message={"server":"","servicePrincipal":"","db":"default","table":"table_ad2a1b8b11fe4111b57b8eeae744cf96","tableType":"MANAGED_TABLE","tableObjJson":"{\"1\":{\"str\":\"table_ad2a1b8b11fe4111b57b8eeae744cf96\"},\"2\":{\"str\":\"default\"},\"3\":{\"str\":\"slave\"},\"4\":{\"i32\":1795143696},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"key\"},\"2\":{\"str\":\"int\"},\"3\":{\"str\":\"\"}}]},\"2\":{\"str\":\"file:/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/warehouse/table_ad2a1b8b11fe4111b57b8eeae744cf96\"},\"3\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"str\":\"\"},\"2\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduSerDe\"},\"3\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"rec\",0]},\"9\":{\"map\":[\"str\",\"str\",7,{\"kudu.table_name\":\"default.table_ad2a1b8b11fe4111b57b8eeae744cf96\",\"kudu.table_id\":\"6c71ec67711e43cd8452336d7b135667\",\"transient_lastDdlTime\":\"1795143696\",\"comment\":\"\",\"kudu.master_addresses\":\"127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751\",\"storage_handler\":\"org.apache.hadoop.hive.kudu.KuduStorageHandler\",\"kudu.cluster_id\":\"7f28cd38e32845a18797274c93499aa6\"}]},\"10\":{\"str\":\"\"},\"11\":{\"str\":\"\"},\"12\":{\"str\":\"MANAGED_TABLE\"},\"14\":{\"tf\":0},\"17\":{\"str\":\"hive\"},\"18\":{\"i32\":1}}","timestamp":1795143696,"files":[]}, messageFormat=json-0.2, catName=hive);
*** Check failure stack trace: ***
I20261120 03:01:41.136466  3323 mvcc.cc:204] Tried to move back new op lower bound from 7352908587889881088 to 7352908578823405568. Current Snapshot: MvccSnapshot[applied={T|T < 7352908587889881088 or (T in {7352908587889881088})}]
    @     0x7f12539ea920  google::LogMessage::Flush() at ??:0
I20261120 03:01:41.143793  2432 catalog_manager.cc:6151] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09: Sending DeleteTablet for 3 replicas of tablet 1d00a31328c245a88512188ff5f3e584
I20261120 03:01:41.147959  2432 catalog_manager.cc:6952] ReplaceTablet: tablet 1d00a31328c245a88512188ff5f3e584 deleted and replaced by tablet 9975a8096d0d4f9aa659a4d8af0ff0d9
I20261120 03:01:41.177425  2983 tablet_service.cc:1542] Processing DeleteTablet for tablet 1d00a31328c245a88512188ff5f3e584 with delete_type TABLET_DATA_DELETED (replaced by tablet 9975a8096d0d4f9aa659a4d8af0ff0d9) from {username='slave'} at 127.0.0.1:49590
    @     0x7f12539ea68f  google::LogMessage::~LogMessage() at ??:0
I20261120 03:01:41.187963  2693 tablet_service.cc:1542] Processing DeleteTablet for tablet 1d00a31328c245a88512188ff5f3e584 with delete_type TABLET_DATA_DELETED (replaced by tablet 9975a8096d0d4f9aa659a4d8af0ff0d9) from {username='slave'} at 127.0.0.1:38244
I20261120 03:01:41.198261  3340 tablet_replica.cc:333] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd: stopping tablet replica
I20261120 03:01:41.199088  3340 raft_consensus.cc:2440] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:41.199656  3340 raft_consensus.cc:2469] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:41.202795  2983 tablet_service.cc:1495] Processing CreateTablet for tablet 9975a8096d0d4f9aa659a4d8af0ff0d9 (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:41.204046  2983 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9975a8096d0d4f9aa659a4d8af0ff0d9. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:41.203687  2693 tablet_service.cc:1495] Processing CreateTablet for tablet 9975a8096d0d4f9aa659a4d8af0ff0d9 (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:41.205005  2693 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9975a8096d0d4f9aa659a4d8af0ff0d9. 1 dirs total, 0 dirs full, 0 dirs failed
W20261120 03:01:41.209759  2941 consensus_peers.cc:597] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f -> Peer d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769): Couldn't send request to peer d9ed1d903ef340ac8f1025c2b7df9bdd. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: STOPPING. This is attempt 1: this message will repeat every 5th retry.
I20261120 03:01:41.210114  3338 tablet_replica.cc:333] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f: stopping tablet replica
I20261120 03:01:41.210922  3338 raft_consensus.cc:2440] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:01:41.211925  3338 raft_consensus.cc:2469] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:41.213898  3338 ts_tablet_manager.cc:1962] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:41.219790  3340 ts_tablet_manager.cc:1962] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting tablet data with delete state TABLET_DATA_DELETED
    @     0x7f125d0ea3ac  kudu::master::HmsNotificationLogListenerTask::Poll() at ??:0
I20261120 03:01:41.238626  3342 tablet_bootstrap.cc:492] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f: Bootstrap starting.
I20261120 03:01:41.240042  2843 tablet_service.cc:1542] Processing DeleteTablet for tablet 1d00a31328c245a88512188ff5f3e584 with delete_type TABLET_DATA_DELETED (replaced by tablet 9975a8096d0d4f9aa659a4d8af0ff0d9) from {username='slave'} at 127.0.0.1:43660
I20261120 03:01:41.240595  2841 tablet_service.cc:1495] Processing CreateTablet for tablet 9975a8096d0d4f9aa659a4d8af0ff0d9 (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:41.241693  2841 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 9975a8096d0d4f9aa659a4d8af0ff0d9. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:41.245258  3342 tablet_bootstrap.cc:654] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:41.248817  3343 tablet_replica.cc:333] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c: stopping tablet replica
I20261120 03:01:41.249604  3343 raft_consensus.cc:2440] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:41.250126  3343 raft_consensus.cc:2469] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:41.252086  3343 ts_tablet_manager.cc:1962] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:41.260979  3342 tablet_bootstrap.cc:492] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f: No bootstrap required, opened a new log
I20261120 03:01:41.261430  3342 ts_tablet_manager.cc:1449] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f: Time spent bootstrapping tablet: real 0.028s	user 0.000s	sys 0.010s
I20261120 03:01:41.263921  3052 heartbeater.cc:345] Connected to a master server at 127.0.96.126:46565
I20261120 03:01:41.264487  3052 heartbeater.cc:500] Master 127.0.96.126:46565 was elected leader, sending a full tablet report...
I20261120 03:01:41.264320  3342 raft_consensus.cc:366] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
I20261120 03:01:41.265055  3342 raft_consensus.cc:392] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:41.265324  3342 raft_consensus.cc:787] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c393ca4a78704704bc9b58916a81fc3f, State: Initialized, Role: FOLLOWER
I20261120 03:01:41.266237  3342 consensus_queue.cc:261] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
    @     0x7f125d0e9b6c  kudu::master::HmsNotificationLogListenerTask::RunLoop() at ??:0
I20261120 03:01:41.272115  3051 heartbeater.cc:500] Master 127.0.96.125:36419 was elected leader, sending a full tablet report...
I20261120 03:01:41.273277  3255 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" instance_seqno: 1795143695668602) as {username='slave'} at 127.0.96.67:48427; Asking this server to re-register.
I20261120 03:01:41.276711  3052 heartbeater.cc:462] Registering TS with master...
W20261120 03:01:41.278800  2432 catalog_manager.cc:5689] Ignoring report from unknown tablet 7c1a821b612b4325bc34a3a16c284b2d
I20261120 03:01:41.281469  3255 ts_manager.cc:210] Registered new tserver with Master: c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993)
I20261120 03:01:41.281550  2432 catalog_manager.cc:5856] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f reported cstate change: term changed from 0 to 1, leader changed from <none> to c393ca4a78704704bc9b58916a81fc3f (127.0.96.67). New cstate: current_term: 1 leader_uuid: "c393ca4a78704704bc9b58916a81fc3f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } health_report { overall_health: HEALTHY } } }
I20261120 03:01:41.285100  2432 catalog_manager.cc:5856] T e61b8f296bad46e4bac632f7e784633b P c393ca4a78704704bc9b58916a81fc3f reported cstate change: term changed from 0 to 1, leader changed from <none> to 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66). New cstate: current_term: 1 leader_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } }
I20261120 03:01:41.289062  3342 ts_tablet_manager.cc:1480] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f: Time spent starting tablet: real 0.027s	user 0.007s	sys 0.000s
    @     0x7f125d0edbe2  kudu::master::HmsNotificationLogListenerTask::Init()::$_0::operator()() at ??:0
I20261120 03:01:41.304037  3343 ts_tablet_manager.cc:1975] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:41.304452  3343 log.cc:1199] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/1d00a31328c245a88512188ff5f3e584
I20261120 03:01:41.305343  3343 ts_tablet_manager.cc:1996] T 1d00a31328c245a88512188ff5f3e584 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting consensus metadata
I20261120 03:01:41.308516  2417 catalog_manager.cc:5081] TS 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): tablet 1d00a31328c245a88512188ff5f3e584 (table default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]) successfully deleted
I20261120 03:01:41.319394  3344 tablet_bootstrap.cc:492] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd: Bootstrap starting.
    @     0x7f125d0edb9a  _ZNSt3__18__invokeIRZN4kudu6master30HmsNotificationLogListenerTask4InitEvE3$_0JEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOS6_DpOS7_ at ??:0
I20261120 03:01:41.327099  3346 tablet_bootstrap.cc:492] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c: Bootstrap starting.
I20261120 03:01:41.341363  3344 tablet_bootstrap.cc:654] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:41.347852  3346 tablet_bootstrap.cc:654] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:41.352121  3340 ts_tablet_manager.cc:1975] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:41.352705  3340 log.cc:1199] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/1d00a31328c245a88512188ff5f3e584
I20261120 03:01:41.353303  3338 ts_tablet_manager.cc:1975] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:41.354338  3340 ts_tablet_manager.cc:1996] T 1d00a31328c245a88512188ff5f3e584 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting consensus metadata
I20261120 03:01:41.354712  3338 log.cc:1199] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/1d00a31328c245a88512188ff5f3e584
I20261120 03:01:41.355520  3338 ts_tablet_manager.cc:1996] T 1d00a31328c245a88512188ff5f3e584 P c393ca4a78704704bc9b58916a81fc3f: Deleting consensus metadata
    @     0x7f125d0edb2a  std::__1::__invoke_void_return_wrapper<>::__call<>() at ??:0
I20261120 03:01:41.355834  2432 master_service.cc:965] ReplaceTablet: received request to replace tablet 1d00a31328c245a88512188ff5f3e584 from {username='slave'} at 127.0.0.1:59970
I20261120 03:01:41.355259  3344 tablet_bootstrap.cc:492] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd: No bootstrap required, opened a new log
I20261120 03:01:41.356441  3344 ts_tablet_manager.cc:1449] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent bootstrapping tablet: real 0.037s	user 0.010s	sys 0.007s
I20261120 03:01:41.358882  2419 catalog_manager.cc:5081] TS c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993): tablet 1d00a31328c245a88512188ff5f3e584 (table default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]) successfully deleted
I20261120 03:01:41.360915  3344 raft_consensus.cc:366] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
I20261120 03:01:41.361675  3344 raft_consensus.cc:392] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:41.361953  3344 raft_consensus.cc:787] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9ed1d903ef340ac8f1025c2b7df9bdd, State: Initialized, Role: FOLLOWER
I20261120 03:01:41.362661  3344 consensus_queue.cc:261] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
I20261120 03:01:41.365181  2419 catalog_manager.cc:5081] TS d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769): tablet 1d00a31328c245a88512188ff5f3e584 (table default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]) successfully deleted
I20261120 03:01:41.365239  2761 heartbeater.cc:500] Master 127.0.96.125:36419 was elected leader, sending a full tablet report...
I20261120 03:01:41.369916  3344 ts_tablet_manager.cc:1480] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent starting tablet: real 0.013s	user 0.004s	sys 0.003s
W20261120 03:01:41.371076  2432 catalog_manager.cc:5689] Ignoring report from unknown tablet 7c1a821b612b4325bc34a3a16c284b2d
I20261120 03:01:41.382869  2425 master_service.cc:965] ReplaceTablet: received request to replace tablet 7c1a821b612b4325bc34a3a16c284b2d from {username='slave'} at 127.0.0.1:59970
I20261120 03:01:41.394125  3346 tablet_bootstrap.cc:492] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c: No bootstrap required, opened a new log
I20261120 03:01:41.394585  3346 ts_tablet_manager.cc:1449] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent bootstrapping tablet: real 0.068s	user 0.017s	sys 0.008s
I20261120 03:01:41.397271  3346 raft_consensus.cc:366] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
I20261120 03:01:41.397948  3346 raft_consensus.cc:392] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:41.398267  3346 raft_consensus.cc:787] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23741ccd042d4ca49afc3b8d1e26e22c, State: Initialized, Role: FOLLOWER
I20261120 03:01:41.399050  3346 consensus_queue.cc:261] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
I20261120 03:01:41.401120  3346 ts_tablet_manager.cc:1480] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent starting tablet: real 0.006s	user 0.005s	sys 0.000s
I20261120 03:01:41.406953  2909 heartbeater.cc:500] Master 127.0.96.125:36419 was elected leader, sending a full tablet report...
    @     0x7f125d0edaf2  std::__1::__function::__alloc_func<>::operator()() at ??:0
    @     0x7f125d0ecdee  std::__1::__function::__func<>::operator()() at ??:0
    @           0x327655  std::__1::__function::__value_func<>::operator()() at /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/../sanitizer_common/sanitizer_vector.h:111
    @           0x32747a  std::__1::function<>::operator()() at /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp:294
W20261120 03:01:41.443497  2432 catalog_manager.cc:5689] Ignoring report from unknown tablet 7c1a821b612b4325bc34a3a16c284b2d
    @     0x7f12540fc7db  kudu::Thread::SuperviseThread() at ??:0
    @           0x277de0  __tsan_thread_start_func at ??:0
    @     0x7f125586a6db  start_thread at ??:0
    @     0x7f1250ba061f  clone at ??:0
I20261120 03:01:41.504505  3238 raft_consensus.cc:534] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:41.504959  3238 raft_consensus.cc:556] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
I20261120 03:01:41.507040  3238 leader_election.cc:290] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993), d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769)
I20261120 03:01:41.508909  2714 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9975a8096d0d4f9aa659a4d8af0ff0d9" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" is_pre_election: true
I20261120 03:01:41.508910  3004 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9975a8096d0d4f9aa659a4d8af0ff0d9" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c393ca4a78704704bc9b58916a81fc3f" is_pre_election: true
I20261120 03:01:41.509603  3004 raft_consensus.cc:2062] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 23741ccd042d4ca49afc3b8d1e26e22c in term 0
I20261120 03:01:41.509603  2714 raft_consensus.cc:2062] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 23741ccd042d4ca49afc3b8d1e26e22c in term 0
I20261120 03:01:41.510808  2799 leader_election.cc:304] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 23741ccd042d4ca49afc3b8d1e26e22c, d9ed1d903ef340ac8f1025c2b7df9bdd; no voters: 
I20261120 03:01:41.511911  3238 raft_consensus.cc:3024] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:41.512164  3238 raft_consensus.cc:534] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:41.512439  3238 raft_consensus.cc:3296] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:41.513775  3238 raft_consensus.cc:556] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
I20261120 03:01:41.519349  3238 leader_election.cc:290] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 election: Requested vote from peers c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993), d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769)
I20261120 03:01:41.520601  2714 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9975a8096d0d4f9aa659a4d8af0ff0d9" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd"
I20261120 03:01:41.520655  3004 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "9975a8096d0d4f9aa659a4d8af0ff0d9" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c393ca4a78704704bc9b58916a81fc3f"
I20261120 03:01:41.521170  3004 raft_consensus.cc:3296] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:41.521157  2714 raft_consensus.cc:3296] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:41.522045  2714 raft_consensus.cc:2062] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 23741ccd042d4ca49afc3b8d1e26e22c in term 1
I20261120 03:01:41.522045  3004 raft_consensus.cc:2062] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 23741ccd042d4ca49afc3b8d1e26e22c in term 1
I20261120 03:01:41.529598  2799 leader_election.cc:304] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 23741ccd042d4ca49afc3b8d1e26e22c, d9ed1d903ef340ac8f1025c2b7df9bdd; no voters: 
I20261120 03:01:41.530609  3238 raft_consensus.cc:3024] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:41.531049  3238 raft_consensus.cc:744] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 LEADER]: Becoming Leader. State: Replica: 23741ccd042d4ca49afc3b8d1e26e22c, State: Running, Role: LEADER
I20261120 03:01:41.531812  3238 consensus_queue.cc:238] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
I20261120 03:01:41.542030  3238 consensus_queue.cc:1049] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [LEADER]: Connected to new peer: Peer: permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:41.545708  3362 consensus_queue.cc:1049] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
W20261120 03:01:41.767163  3060 connection.cc:573] client connection to 127.0.96.125:36419 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:41.767174  2938 connection.cc:573] client connection to 127.0.96.125:36419 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:41.768756  3060 master_proxy_rpc.cc:203] Re-attempting ReplaceTablet request to leader Master (127.0.96.125:36419)
W20261120 03:01:41.769395  3060 master_proxy_rpc.cc:203] Re-attempting AlterTable request to leader Master (127.0.96.125:36419)
W20261120 03:01:41.770279  3060 master_proxy_rpc.cc:203] Re-attempting CreateTable request to leader Master (127.0.96.125:36419)
W20261120 03:01:41.770545  2795 connection.cc:573] client connection to 127.0.96.125:36419 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:41.771129  3060 master_proxy_rpc.cc:203] Re-attempting DeleteTable request to leader Master (127.0.96.125:36419)
W20261120 03:01:41.773957  2648 connection.cc:573] client connection to 127.0.96.125:36419 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:41.777055  3121 negotiation.cc:336] Failed RPC negotiation. Trace:
1120 03:01:41.768922 (+     0us) reactor.cc:730] Submitting negotiation task for client connection to 127.0.96.125:36419 (local address 127.0.96.67:40975)
1120 03:01:41.769866 (+   944us) negotiation.cc:107] Waiting for socket to connect
1120 03:01:41.769909 (+    43us) client_negotiation.cc:175] Beginning negotiation
1120 03:01:41.770181 (+   272us) client_negotiation.cc:262] Sending NEGOTIATE NegotiatePB request
1120 03:01:41.777007 (+  6826us) negotiation.cc:326] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.96.125:36419: BlockingRecv error: recv error from unknown peer: Connection reset by peer (error 104)
Metrics: {"client-negotiator.queue_time_us":347}
W20261120 03:01:41.794507  3139 negotiation.cc:336] Failed RPC negotiation. Trace:
1120 03:01:41.772309 (+     0us) reactor.cc:730] Submitting negotiation task for client connection to 127.0.96.125:36419 (local address 127.0.96.66:41705)
1120 03:01:41.773242 (+   933us) negotiation.cc:107] Waiting for socket to connect
1120 03:01:41.773285 (+    43us) client_negotiation.cc:175] Beginning negotiation
1120 03:01:41.773529 (+   244us) client_negotiation.cc:262] Sending NEGOTIATE NegotiatePB request
1120 03:01:41.794459 (+ 20930us) negotiation.cc:326] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.96.125:36419: BlockingRecv error: recv error from unknown peer: Connection reset by peer (error 104)
Metrics: {"client-negotiator.queue_time_us":343}
W20261120 03:01:41.796648  3124 negotiation.cc:336] Failed RPC negotiation. Trace:
1120 03:01:41.775716 (+     0us) reactor.cc:730] Submitting negotiation task for client connection to 127.0.96.125:36419 (local address 127.0.96.65:44919)
1120 03:01:41.776356 (+   640us) negotiation.cc:107] Waiting for socket to connect
1120 03:01:41.776398 (+    42us) client_negotiation.cc:175] Beginning negotiation
1120 03:01:41.776632 (+   234us) client_negotiation.cc:262] Sending NEGOTIATE NegotiatePB request
1120 03:01:41.796587 (+ 19955us) negotiation.cc:326] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.96.125:36419: BlockingRecv error: recv error from unknown peer: Connection reset by peer (error 104)
Metrics: {"client-negotiator.queue_time_us":83}
W20261120 03:01:41.796691  3372 negotiation.cc:336] Failed RPC negotiation. Trace:
1120 03:01:41.772756 (+     0us) reactor.cc:730] Submitting negotiation task for client connection to 127.0.96.125:36419 (local address 127.0.0.1:60958)
1120 03:01:41.788581 (+ 15825us) negotiation.cc:107] Waiting for socket to connect
1120 03:01:41.788635 (+    54us) client_negotiation.cc:175] Beginning negotiation
1120 03:01:41.788942 (+   307us) client_negotiation.cc:262] Sending NEGOTIATE NegotiatePB request
1120 03:01:41.796620 (+  7678us) negotiation.cc:326] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.0.96.125:36419: BlockingRecv error: recv error from unknown peer: Connection reset by peer (error 104)
Metrics: {"client-negotiator.queue_time_us":15146,"thread_start_us":16292,"threads_started":1}
I20261120 03:01:41.831151  3374 raft_consensus.cc:534] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader a23f052846654c9daf3fb0d90fbb6f09)
I20261120 03:01:41.832849  3374 raft_consensus.cc:556] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
W20261120 03:01:41.863940  3240 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.125:36419: connect: Connection refused (error 111)
I20261120 03:01:41.865732  3374 leader_election.cc:290] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:01:41.890216  3240 leader_election.cc:336] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419): Network error: Client connection negotiation failed: client connection to 127.0.96.125:36419: connect: Connection refused (error 111)
W20261120 03:01:41.892697  3240 leader_election.cc:336] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:41.893129  3240 leader_election.cc:304] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6e18f436ef9945d78f543c3471320ee7; no voters: 86ed298871604dbe84645c0199254db8, a23f052846654c9daf3fb0d90fbb6f09
I20261120 03:01:41.893889  3374 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20261120 03:01:42.605154  3382 raft_consensus.cc:534] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader a23f052846654c9daf3fb0d90fbb6f09)
I20261120 03:01:42.605762  3382 raft_consensus.cc:556] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:42.610240  3382 leader_election.cc:290] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:01:42.616549  3240 leader_election.cc:336] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419): Network error: Client connection negotiation failed: client connection to 127.0.96.125:36419: connect: Connection refused (error 111)
W20261120 03:01:42.617224  3240 leader_election.cc:336] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:42.617655  3240 leader_election.cc:304] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6e18f436ef9945d78f543c3471320ee7; no voters: 86ed298871604dbe84645c0199254db8, a23f052846654c9daf3fb0d90fbb6f09
I20261120 03:01:42.618394  3382 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20261120 03:01:43.394462  3394 raft_consensus.cc:534] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader a23f052846654c9daf3fb0d90fbb6f09)
I20261120 03:01:43.394956  3394 raft_consensus.cc:556] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:43.396552  3394 leader_election.cc:290] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:01:43.401243  3240 leader_election.cc:336] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419): Network error: Client connection negotiation failed: client connection to 127.0.96.125:36419: connect: Connection refused (error 111)
W20261120 03:01:43.401921  3240 leader_election.cc:336] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:43.402269  3240 leader_election.cc:304] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6e18f436ef9945d78f543c3471320ee7; no voters: 86ed298871604dbe84645c0199254db8, a23f052846654c9daf3fb0d90fbb6f09
I20261120 03:01:43.402812  3394 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
I20261120 03:01:43.980084   385 external_mini_cluster.cc:1372] Running /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
/tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/wal
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/logs
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.125:36419
--webserver_interface=127.0.96.125
--webserver_port=37771
--builtin_ntp_servers=127.0.96.84:36533
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751
--hive_metastore_uris=thrift://127.0.0.1:45151
--hive_metastore_notification_log_poll_period_seconds=1
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--catalog_manager_bg_task_wait_ms=10
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--leader_failure_max_missed_heartbeat_periods=1.0
--log_preallocate_segments=false with env {}
W20261120 03:01:44.315182  3400 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
W20261120 03:01:44.315763  3400 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20261120 03:01:44.316231  3400 flags.cc:432] Enabled unsafe flag: --never_fsync=true
W20261120 03:01:44.352828  3400 flags.cc:432] Enabled experimental flag: --enable_metadata_cleanup_for_deleted_tables_and_tablets=true
W20261120 03:01:44.353101  3400 flags.cc:432] Enabled experimental flag: --metadata_for_deleted_table_and_tablet_reserved_secs=0
W20261120 03:01:44.353323  3400 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
W20261120 03:01:44.353533  3400 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
W20261120 03:01:44.353749  3400 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
W20261120 03:01:44.353958  3400 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
I20261120 03:01:44.393689  3400 master_runner.cc:387] Master server non-default flags:
--log_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/logs
--logbuflevel=-1
--logtostderr=true
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.0.96.84:36533
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--log_preallocate_segments=false
--leader_failure_max_missed_heartbeat_periods=1
--fs_data_dirs=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/data
--fs_wal_dir=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/wal
--hive_metastore_uris=thrift://127.0.0.1:45151
--catalog_manager_bg_task_wait_ms=10
--enable_metadata_cleanup_for_deleted_tables_and_tablets=true
--metadata_for_deleted_table_and_tablet_reserved_secs=0
--hive_metastore_notification_log_poll_period_seconds=1
--ipki_ca_key_size=768
--master_addresses=127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.96.125:36419
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/data/info.pb
--webserver_interface=127.0.96.125
--webserver_port=37771
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false

Master server version:
kudu 1.19.0-SNAPSHOT
revision 36b6772d828145f8fbba3c49746c86838683f9ba
build type FASTDEBUG
built by None at 20 Nov 2026 02:45:53 UTC on bdcb31816ec0
build id 14870
TSAN enabled
I20261120 03:01:44.395358  3400 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20261120 03:01:44.397101  3400 file_cache.cc:504] Constructed file cache file cache with capacity 419430
W20261120 03:01:44.410416  3410 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:01:45.081169  3417 raft_consensus.cc:534] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader a23f052846654c9daf3fb0d90fbb6f09)
I20261120 03:01:45.081557  3417 raft_consensus.cc:556] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:45.083077  3417 leader_election.cc:290] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:01:45.088562  3240 leader_election.cc:336] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419): Network error: Client connection negotiation failed: client connection to 127.0.96.125:36419: connect: Connection refused (error 111)
W20261120 03:01:45.089407  3240 leader_election.cc:336] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:45.089833  3240 leader_election.cc:304] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6e18f436ef9945d78f543c3471320ee7; no voters: 86ed298871604dbe84645c0199254db8, a23f052846654c9daf3fb0d90fbb6f09
I20261120 03:01:45.090590  3417 raft_consensus.cc:2966] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
W20261120 03:01:44.411974  3411 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:01:44.414817  3400 server_base.cc:1061] running on GCE node
W20261120 03:01:45.553715  3414 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20261120 03:01:45.556324  3400 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20261120 03:01:45.559101  3400 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20261120 03:01:45.560462  3400 hybrid_clock.cc:648] HybridClock initialized: now 1795143705560436 us; error 32 us; skew 500 ppm
I20261120 03:01:45.561357  3400 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20261120 03:01:45.568336  3400 webserver.cc:577] Webserver started at http://127.0.96.125:37771/ using document root <none> and password file <none>
I20261120 03:01:45.569274  3400 fs_manager.cc:362] Metadata directory not provided
I20261120 03:01:45.569494  3400 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20261120 03:01:45.577508  3400 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.006s	sys 0.000s
I20261120 03:01:45.582209  3430 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
I20261120 03:01:45.583320  3400 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.001s	sys 0.003s
I20261120 03:01:45.583624  3400 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/data,/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/wal
uuid: "a23f052846654c9daf3fb0d90fbb6f09"
format_stamp: "Formatted at 2026-11-20 03:01:25 on dist-test-slave-jpwh"
I20261120 03:01:45.585527  3400 fs_report.cc:389] FS layout report
--------------------
wal directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/wal
metadata directory: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/wal
1 data directories: /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20261120 03:01:45.632781  3400 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
I20261120 03:01:45.634363  3400 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20261120 03:01:45.634784  3400 kserver.cc:163] Server-wide thread pool size limit: 3276
I20261120 03:01:45.710217  3400 rpc_server.cc:307] RPC server started. Bound to: 127.0.96.125:36419
I20261120 03:01:45.710268  3481 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.96.125:36419 every 8 connection(s)
I20261120 03:01:45.713373  3400 server_base.cc:1193] Dumped server information to /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/master-1/data/info.pb
I20261120 03:01:45.718708   385 external_mini_cluster.cc:1434] Started /tmp/dist-test-tasklxEiy3/build/tsan/bin/kudu as pid 3400
I20261120 03:01:45.730440  3482 sys_catalog.cc:263] Verifying existing consensus state
I20261120 03:01:45.739233  3482 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09: Bootstrap starting.
I20261120 03:01:45.775225  3482 log.cc:826] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09: Log is configured to *not* fsync() on all Append() calls
I20261120 03:01:45.831835  2761 heartbeater.cc:345] Connected to a master server at 127.0.96.125:36419
I20261120 03:01:45.854936  3482 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09: Bootstrap replayed 1/1 log segments. Stats: ops{read=28 overwritten=0 applied=28 ignored=0} inserts{seen=16 ignored=0} mutations{seen=26 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20261120 03:01:45.855643  3482 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09: Bootstrap complete.
I20261120 03:01:45.874342  3482 raft_consensus.cc:366] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:45.876447  3482 raft_consensus.cc:787] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: a23f052846654c9daf3fb0d90fbb6f09, State: Initialized, Role: FOLLOWER
I20261120 03:01:45.877225  3482 consensus_queue.cc:261] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 28, Last appended: 3.28, Last appended by leader: 28, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:45.879984  3486 sys_catalog.cc:455] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:45.880831  3486 sys_catalog.cc:458] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:01:45.881038  3482 sys_catalog.cc:565] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: configured and running, proceeding with master startup.
I20261120 03:01:45.928776  3501 catalog_manager.cc:1310] Loaded cluster ID: 7f28cd38e32845a18797274c93499aa6
I20261120 03:01:45.929168  3501 catalog_manager.cc:1603] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09: loading cluster ID for follower catalog manager: success
I20261120 03:01:45.939800  3501 catalog_manager.cc:1625] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09: acquiring CA information for follower catalog manager: success
I20261120 03:01:45.946544  3501 catalog_manager.cc:1653] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
I20261120 03:01:45.977757  3051 heartbeater.cc:345] Connected to a master server at 127.0.96.125:36419
I20261120 03:01:45.968457  3400 master_runner.cc:187] Error getting master registration for 127.0.96.124:33751: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111), 
I20261120 03:01:45.990724  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:01:45.992554  3447 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" instance_seqno: 1795143695668602) as {username='slave'} at 127.0.96.67:37345; Asking this server to re-register.
I20261120 03:01:45.994117  3051 heartbeater.cc:462] Registering TS with master...
I20261120 03:01:45.998229  3447 ts_manager.cc:210] Registered new tserver with Master: c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993)
I20261120 03:01:46.117501  3122 master_runner.cc:187] Error getting master registration for 127.0.96.124:33751: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111), 
I20261120 03:01:46.117890  3122 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:01:46.122933  3507 raft_consensus.cc:534] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader a23f052846654c9daf3fb0d90fbb6f09)
I20261120 03:01:46.123277  3507 raft_consensus.cc:556] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:46.124979  3507 leader_election.cc:290] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
I20261120 03:01:46.126586  3457 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6e18f436ef9945d78f543c3471320ee7" candidate_term: 4 candidate_status { last_received { term: 3 index: 28 } } ignore_live_leader: false dest_uuid: "a23f052846654c9daf3fb0d90fbb6f09" is_pre_election: true
I20261120 03:01:46.127614  3457 raft_consensus.cc:2062] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 3 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 6e18f436ef9945d78f543c3471320ee7 in term 3
I20261120 03:01:46.129388  3240 leader_election.cc:304] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6e18f436ef9945d78f543c3471320ee7, a23f052846654c9daf3fb0d90fbb6f09; no voters: 
I20261120 03:01:46.130118  3507 raft_consensus.cc:3024] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Leader pre-election won for term 4
I20261120 03:01:46.130371  3507 raft_consensus.cc:534] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Starting leader election (detected failure of leader a23f052846654c9daf3fb0d90fbb6f09)
I20261120 03:01:46.130625  3507 raft_consensus.cc:3296] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 3 FOLLOWER]: Advancing to term 4
W20261120 03:01:46.131637  3240 leader_election.cc:336] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:46.132210  3507 raft_consensus.cc:556] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:46.138134  3507 leader_election.cc:290] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 election: Requested vote from peers a23f052846654c9daf3fb0d90fbb6f09 (127.0.96.125:36419), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
I20261120 03:01:46.138908  3457 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6e18f436ef9945d78f543c3471320ee7" candidate_term: 4 candidate_status { last_received { term: 3 index: 28 } } ignore_live_leader: false dest_uuid: "a23f052846654c9daf3fb0d90fbb6f09"
I20261120 03:01:46.139441  3457 raft_consensus.cc:3296] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 3 FOLLOWER]: Advancing to term 4
I20261120 03:01:46.140648  3457 raft_consensus.cc:2062] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader election vote request: granting yes vote for candidate 6e18f436ef9945d78f543c3471320ee7 in term 4
W20261120 03:01:46.148955  3240 leader_election.cc:336] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:46.158017  3240 leader_election.cc:304] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6e18f436ef9945d78f543c3471320ee7, a23f052846654c9daf3fb0d90fbb6f09; no voters: 86ed298871604dbe84645c0199254db8
I20261120 03:01:46.159063  3507 raft_consensus.cc:3024] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 4 FOLLOWER]: Leader election won for term 4
I20261120 03:01:46.159536  3507 raft_consensus.cc:744] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [term 4 LEADER]: Becoming Leader. State: Replica: 6e18f436ef9945d78f543c3471320ee7, State: Running, Role: LEADER
I20261120 03:01:46.160408  3507 consensus_queue.cc:238] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 28, Committed index: 28, Last appended: 3.28, Last appended by leader: 28, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:46.162963  2909 heartbeater.cc:345] Connected to a master server at 127.0.96.125:36419
I20261120 03:01:46.166461  3509 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6e18f436ef9945d78f543c3471320ee7. Latest consensus state: current_term: 4 leader_uuid: "6e18f436ef9945d78f543c3471320ee7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:46.167313  3509 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: This master's current role is: LEADER
I20261120 03:01:46.167732  3447 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" instance_seqno: 1795143693763750) as {username='slave'} at 127.0.96.66:52685; Asking this server to re-register.
I20261120 03:01:46.170274  2909 heartbeater.cc:462] Registering TS with master...
I20261120 03:01:46.173820  3457 raft_consensus.cc:1338] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Refusing update from remote peer 6e18f436ef9945d78f543c3471320ee7: Log matching property violated. Preceding OpId in replica: term: 3 index: 28. Preceding OpId from leader: term: 4 index: 29. (index mismatch)
I20261120 03:01:46.175020  3447 ts_manager.cc:210] Registered new tserver with Master: 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733)
I20261120 03:01:46.176653  3513 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
I20261120 03:01:46.176904  3512 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 29, Last known committed idx: 28, Time since last communication: 0.000s
W20261120 03:01:46.177876  3240 consensus_peers.cc:597] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 -> Peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Couldn't send request to peer 86ed298871604dbe84645c0199254db8. Status: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20261120 03:01:46.186884  3513 catalog_manager.cc:694] Loaded metadata for table default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]
I20261120 03:01:46.188575  3513 catalog_manager.cc:694] Loaded metadata for table default.table_a938daa15a8b429190ccb7a61dab9f86 [id=957365c9d24645e985a308158ec0af66]
I20261120 03:01:46.190526  3513 catalog_manager.cc:694] Loaded metadata for table default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb]
I20261120 03:01:46.190631  3486 sys_catalog.cc:455] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6e18f436ef9945d78f543c3471320ee7. Latest consensus state: current_term: 4 leader_uuid: "6e18f436ef9945d78f543c3471320ee7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:46.191565  3486 sys_catalog.cc:458] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:01:46.200286  3513 tablet_loader.cc:96] loaded metadata for tablet 9975a8096d0d4f9aa659a4d8af0ff0d9 (table default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3])
I20261120 03:01:46.201733  3513 tablet_loader.cc:96] loaded metadata for tablet 9cc585adb4df48d18b69d8102382bf2e (table default.table_a938daa15a8b429190ccb7a61dab9f86 [id=957365c9d24645e985a308158ec0af66])
I20261120 03:01:46.202349  3514 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "6e18f436ef9945d78f543c3471320ee7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:46.203048  3513 tablet_loader.cc:96] loaded metadata for tablet e61b8f296bad46e4bac632f7e784633b (table default.table_1c9622c0bec4498cb886d15294dc5395 [id=982cb6e783c64717b3882765a530c3eb])
I20261120 03:01:46.203126  3514 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7 [sys.catalog]: This master's current role is: LEADER
I20261120 03:01:46.204495  3513 catalog_manager.cc:1535] Initializing Kudu cluster ID...
I20261120 03:01:46.207808  3486 sys_catalog.cc:455] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "6e18f436ef9945d78f543c3471320ee7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } } }
I20261120 03:01:46.208312  3486 sys_catalog.cc:458] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [sys.catalog]: This master's current role is: FOLLOWER
I20261120 03:01:46.209635  3513 catalog_manager.cc:1310] Loaded cluster ID: 7f28cd38e32845a18797274c93499aa6
I20261120 03:01:46.209882  3513 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
I20261120 03:01:46.215306  3513 catalog_manager.cc:1555] Loading token signing keys...
I20261120 03:01:46.220137  3513 catalog_manager.cc:6248] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: Loaded TSK: 0
I20261120 03:01:46.221382  3513 catalog_manager.cc:1565] Initializing in-progress tserver states...
I20261120 03:01:46.226109  3513 catalog_manager.cc:1578] Loading latest processed Hive Metastore notification log event ID...
I20261120 03:01:46.230731  3513 catalog_manager.cc:6086] Last processed Hive Metastore notification event ID: -1
I20261120 03:01:46.260905  3255 master_service.cc:965] ReplaceTablet: received request to replace tablet 9975a8096d0d4f9aa659a4d8af0ff0d9 from {username='slave'} at 127.0.0.1:35266
I20261120 03:01:46.261917  3254 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:35266:
table { table_name: "default.Table_e387307c289c48c392b4a49829f76842" } new_table_name: "default.Table_11fd9a471c414923bd22c7180c831f74" modify_external_catalogs: true
I20261120 03:01:46.262790  3252 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35266:
name: "default.Table_8feab37db9ac453aaa1afb4721505818"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:46.263240  3256 catalog_manager.cc:3588] Servicing AlterTable request from {username='slave'} at 127.0.0.1:35266:
table { table_name: "default.Table_1c9622c0bec4498cb886d15294dc5395" } new_table_name: "default.Table_b9ef32f90b37407999140f1090dc8049" modify_external_catalogs: true
I20261120 03:01:46.263482  3253 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35266:
name: "default.Table_4eedec7d9c82426698cb181fa17e7adb"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:46.265195  3251 catalog_manager.cc:2551] Servicing SoftDeleteTable request from {username='slave'} at 127.0.0.1:35266:
table { table_name: "default.Table_a938daa15a8b429190ccb7a61dab9f86" } modify_external_catalogs: true
I20261120 03:01:46.265465  3250 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35266:
name: "default.Table_35bb29085419456eb14b37a3b5c7e92a"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:46.265919  3251 catalog_manager.cc:2799] Servicing DeleteTable request from {username='slave'} at 127.0.0.1:35266:
table { table_name: "default.Table_a938daa15a8b429190ccb7a61dab9f86" } modify_external_catalogs: true
I20261120 03:01:46.269085  3249 master_service.cc:965] ReplaceTablet: received request to replace tablet 9cc585adb4df48d18b69d8102382bf2e from {username='slave'} at 127.0.0.1:35266
F20261120 03:01:46.277612  3316 hms_notification_log_listener.cc:287] non-monotonous event IDs from HMS: current 1, previous 1; dumping first 2 out of 4 received events: NotificationEvent(eventId=1, eventTime=1795143696, eventType=CREATE_TABLE, dbName=default, tableName=table_1c9622c0bec4498cb886d15294dc5395, message={"server":"","servicePrincipal":"","db":"default","table":"table_1c9622c0bec4498cb886d15294dc5395","tableType":"MANAGED_TABLE","tableObjJson":"{\"1\":{\"str\":\"table_1c9622c0bec4498cb886d15294dc5395\"},\"2\":{\"str\":\"default\"},\"3\":{\"str\":\"slave\"},\"4\":{\"i32\":1795143696},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"key\"},\"2\":{\"str\":\"int\"},\"3\":{\"str\":\"\"}}]},\"2\":{\"str\":\"file:/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/warehouse/table_1c9622c0bec4498cb886d15294dc5395\"},\"3\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"str\":\"\"},\"2\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduSerDe\"},\"3\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"rec\",0]},\"9\":{\"map\":[\"str\",\"str\",7,{\"kudu.table_name\":\"default.table_1c9622c0bec4498cb886d15294dc5395\",\"kudu.table_id\":\"982cb6e783c64717b3882765a530c3eb\",\"transient_lastDdlTime\":\"1795143696\",\"comment\":\"\",\"kudu.master_addresses\":\"127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751\",\"storage_handler\":\"org.apache.hadoop.hive.kudu.KuduStorageHandler\",\"kudu.cluster_id\":\"7f28cd38e32845a18797274c93499aa6\"}]},\"10\":{\"str\":\"\"},\"11\":{\"str\":\"\"},\"12\":{\"str\":\"MANAGED_TABLE\"},\"14\":{\"tf\":0},\"17\":{\"str\":\"hive\"},\"18\":{\"i32\":1}}","timestamp":1795143696,"files":[]}, messageFormat=json-0.2, catName=hive); NotificationEvent(eventId=1, eventTime=1795143696, eventType=CREATE_TABLE, dbName=default, tableName=table_ad2a1b8b11fe4111b57b8eeae744cf96, message={"server":"","servicePrincipal":"","db":"default","table":"table_ad2a1b8b11fe4111b57b8eeae744cf96","tableType":"MANAGED_TABLE","tableObjJson":"{\"1\":{\"str\":\"table_ad2a1b8b11fe4111b57b8eeae744cf96\"},\"2\":{\"str\":\"default\"},\"3\":{\"str\":\"slave\"},\"4\":{\"i32\":1795143696},\"5\":{\"i32\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"lst\":[\"rec\",1,{\"1\":{\"str\":\"key\"},\"2\":{\"str\":\"int\"},\"3\":{\"str\":\"\"}}]},\"2\":{\"str\":\"file:/tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/warehouse/table_ad2a1b8b11fe4111b57b8eeae744cf96\"},\"3\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduInputFormat\"},\"4\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduOutputFormat\"},\"5\":{\"tf\":0},\"6\":{\"i32\":0},\"7\":{\"rec\":{\"1\":{\"str\":\"\"},\"2\":{\"str\":\"org.apache.hadoop.hive.kudu.KuduSerDe\"},\"3\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"str\",0]},\"9\":{\"lst\":[\"rec\",0]},\"10\":{\"map\":[\"str\",\"str\",0,{}]}}},\"8\":{\"lst\":[\"rec\",0]},\"9\":{\"map\":[\"str\",\"str\",7,{\"kudu.table_name\":\"default.table_ad2a1b8b11fe4111b57b8eeae744cf96\",\"kudu.table_id\":\"6c71ec67711e43cd8452336d7b135667\",\"transient_lastDdlTime\":\"1795143696\",\"comment\":\"\",\"kudu.master_addresses\":\"127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751\",\"storage_handler\":\"org.apache.hadoop.hive.kudu.KuduStorageHandler\",\"kudu.cluster_id\":\"7f28cd38e32845a18797274c93499aa6\"}]},\"10\":{\"str\":\"\"},\"11\":{\"str\":\"\"},\"12\":{\"str\":\"MANAGED_TABLE\"},\"14\":{\"tf\":0},\"17\":{\"str\":\"hive\"},\"18\":{\"i32\":1}}","timestamp":1795143696,"files":[]}, messageFormat=json-0.2, catName=hive);
*** Check failure stack trace: ***
I20261120 03:01:46.277925  3248 catalog_manager.cc:2301] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35266:
name: "default.Table_ef2c937cb5564121ad7b40189665d8ba"
schema {
  columns {
    name: "key"
    type: INT32
    is_key: true
    is_nullable: false
    encoding: AUTO_ENCODING
    compression: DEFAULT_COMPRESSION
    cfile_block_size: 0
    immutable: false
  }
}
split_rows_range_bounds {
}
partition_schema {
  range_schema {
    columns {
      name: "key"
    }
  }
}
I20261120 03:01:46.290333  3255 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: Sending DeleteTablet for 3 replicas of tablet 9975a8096d0d4f9aa659a4d8af0ff0d9
I20261120 03:01:46.292035  3249 catalog_manager.cc:6151] T 00000000000000000000000000000000 P 6e18f436ef9945d78f543c3471320ee7: Sending DeleteTablet for 3 replicas of tablet 9cc585adb4df48d18b69d8102382bf2e
I20261120 03:01:46.294320  3249 catalog_manager.cc:6952] ReplaceTablet: tablet 9cc585adb4df48d18b69d8102382bf2e deleted and replaced by tablet 7b3a5f32068a44668cb454e6bc317f2a
I20261120 03:01:46.295933  3255 catalog_manager.cc:6952] ReplaceTablet: tablet 9975a8096d0d4f9aa659a4d8af0ff0d9 deleted and replaced by tablet fb3606184a5141b8b0e80ba6902895db
    @     0x7f5ae3e03920  google::LogMessage::Flush() at ??:0
I20261120 03:01:46.339639  2761 heartbeater.cc:345] Connected to a master server at 127.0.96.125:36419
I20261120 03:01:46.344323  3447 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" instance_seqno: 1795143691680008) as {username='slave'} at 127.0.96.65:58691; Asking this server to re-register.
I20261120 03:01:46.346048  2761 heartbeater.cc:462] Registering TS with master...
I20261120 03:01:46.350893  3447 ts_manager.cc:210] Registered new tserver with Master: d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769)
    @     0x7f5ae3e0368f  google::LogMessage::~LogMessage() at ??:0
I20261120 03:01:46.360899  2984 tablet_service.cc:1495] Processing CreateTablet for tablet 7b3a5f32068a44668cb454e6bc317f2a (DEFAULT_TABLE table=default.table_a938daa15a8b429190ccb7a61dab9f86 [id=957365c9d24645e985a308158ec0af66]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:46.361631  2983 tablet_service.cc:1542] Processing DeleteTablet for tablet 9975a8096d0d4f9aa659a4d8af0ff0d9 with delete_type TABLET_DATA_DELETED (replaced by tablet fb3606184a5141b8b0e80ba6902895db) from {username='slave'} at 127.0.0.1:49596
I20261120 03:01:46.362485  2984 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7b3a5f32068a44668cb454e6bc317f2a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:46.362826  2985 tablet_service.cc:1542] Processing DeleteTablet for tablet 9cc585adb4df48d18b69d8102382bf2e with delete_type TABLET_DATA_DELETED (replaced by tablet 7b3a5f32068a44668cb454e6bc317f2a) from {username='slave'} at 127.0.0.1:49596
I20261120 03:01:46.364223  3529 tablet_replica.cc:333] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f: stopping tablet replica
I20261120 03:01:46.365020  3529 raft_consensus.cc:2440] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:46.365666  3529 raft_consensus.cc:2469] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:46.360899  2982 tablet_service.cc:1495] Processing CreateTablet for tablet fb3606184a5141b8b0e80ba6902895db (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:46.368176  3529 ts_tablet_manager.cc:1962] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:46.369145  2982 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fb3606184a5141b8b0e80ba6902895db. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:46.386184  2841 tablet_service.cc:1542] Processing DeleteTablet for tablet 9975a8096d0d4f9aa659a4d8af0ff0d9 with delete_type TABLET_DATA_DELETED (replaced by tablet fb3606184a5141b8b0e80ba6902895db) from {username='slave'} at 127.0.0.1:43676
I20261120 03:01:46.387387  2843 tablet_service.cc:1542] Processing DeleteTablet for tablet 9cc585adb4df48d18b69d8102382bf2e with delete_type TABLET_DATA_DELETED (replaced by tablet 7b3a5f32068a44668cb454e6bc317f2a) from {username='slave'} at 127.0.0.1:43676
I20261120 03:01:46.388878  2842 tablet_service.cc:1495] Processing CreateTablet for tablet 7b3a5f32068a44668cb454e6bc317f2a (DEFAULT_TABLE table=default.table_a938daa15a8b429190ccb7a61dab9f86 [id=957365c9d24645e985a308158ec0af66]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:46.390282  2842 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7b3a5f32068a44668cb454e6bc317f2a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:46.395238  2840 tablet_service.cc:1495] Processing CreateTablet for tablet fb3606184a5141b8b0e80ba6902895db (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:46.396453  2840 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fb3606184a5141b8b0e80ba6902895db. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:46.416532  3529 ts_tablet_manager.cc:1975] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:46.416985  3529 log.cc:1199] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/9975a8096d0d4f9aa659a4d8af0ff0d9
I20261120 03:01:46.417567  2693 tablet_service.cc:1542] Processing DeleteTablet for tablet 9cc585adb4df48d18b69d8102382bf2e with delete_type TABLET_DATA_DELETED (replaced by tablet 7b3a5f32068a44668cb454e6bc317f2a) from {username='slave'} at 127.0.0.1:38248
I20261120 03:01:46.419031  2694 tablet_service.cc:1542] Processing DeleteTablet for tablet 9975a8096d0d4f9aa659a4d8af0ff0d9 with delete_type TABLET_DATA_DELETED (replaced by tablet fb3606184a5141b8b0e80ba6902895db) from {username='slave'} at 127.0.0.1:38248
I20261120 03:01:46.419145  2695 tablet_service.cc:1495] Processing CreateTablet for tablet 7b3a5f32068a44668cb454e6bc317f2a (DEFAULT_TABLE table=default.table_a938daa15a8b429190ccb7a61dab9f86 [id=957365c9d24645e985a308158ec0af66]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:46.420264  2695 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 7b3a5f32068a44668cb454e6bc317f2a. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:46.426923  2692 tablet_service.cc:1495] Processing CreateTablet for tablet fb3606184a5141b8b0e80ba6902895db (DEFAULT_TABLE table=default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]), partition=RANGE (key) PARTITION UNBOUNDED
I20261120 03:01:46.428022  2692 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet fb3606184a5141b8b0e80ba6902895db. 1 dirs total, 0 dirs full, 0 dirs failed
I20261120 03:01:46.429273  3530 tablet_replica.cc:333] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c: stopping tablet replica
I20261120 03:01:46.431349  3529 ts_tablet_manager.cc:1996] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P c393ca4a78704704bc9b58916a81fc3f: Deleting consensus metadata
I20261120 03:01:46.435513  3530 raft_consensus.cc:2440] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:01:46.438143  3529 tablet_replica.cc:333] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f: stopping tablet replica
I20261120 03:01:46.438402  3243 catalog_manager.cc:5081] TS c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993): tablet 9975a8096d0d4f9aa659a4d8af0ff0d9 (table default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]) successfully deleted
I20261120 03:01:46.438966  3532 tablet_replica.cc:333] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd: stopping tablet replica
I20261120 03:01:46.439749  3529 raft_consensus.cc:2440] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [term 1 LEADER]: Raft consensus shutting down.
I20261120 03:01:46.439785  3532 raft_consensus.cc:2440] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd [term 1 FOLLOWER]: Raft consensus shutting down.
    @     0x7f5aed5033ac  kudu::master::HmsNotificationLogListenerTask::Poll() at ??:0
I20261120 03:01:46.440266  3532 raft_consensus.cc:2469] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:46.443899  3530 raft_consensus.cc:2469] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:46.446204  3530 ts_tablet_manager.cc:1962] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:46.448561  3529 raft_consensus.cc:2469] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:46.450742  3529 ts_tablet_manager.cc:1962] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:46.457086  3532 ts_tablet_manager.cc:1962] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:46.457960  3534 tablet_bootstrap.cc:492] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c: Bootstrap starting.
I20261120 03:01:46.470330  3531 tablet_bootstrap.cc:492] T fb3606184a5141b8b0e80ba6902895db P c393ca4a78704704bc9b58916a81fc3f: Bootstrap starting.
I20261120 03:01:46.470993  3535 tablet_bootstrap.cc:492] T fb3606184a5141b8b0e80ba6902895db P d9ed1d903ef340ac8f1025c2b7df9bdd: Bootstrap starting.
I20261120 03:01:46.473618  3534 tablet_bootstrap.cc:654] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:46.481503  3535 tablet_bootstrap.cc:654] T fb3606184a5141b8b0e80ba6902895db P d9ed1d903ef340ac8f1025c2b7df9bdd: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:46.481968  3529 ts_tablet_manager.cc:1975] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:46.487099  3534 tablet_bootstrap.cc:492] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c: No bootstrap required, opened a new log
I20261120 03:01:46.487716  3534 ts_tablet_manager.cc:1449] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent bootstrapping tablet: real 0.030s	user 0.015s	sys 0.003s
    @     0x7f5aed502b6c  kudu::master::HmsNotificationLogListenerTask::RunLoop() at ??:0
I20261120 03:01:46.491861  3534 raft_consensus.cc:366] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
I20261120 03:01:46.492589  3534 raft_consensus.cc:392] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:46.492841  3534 raft_consensus.cc:787] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23741ccd042d4ca49afc3b8d1e26e22c, State: Initialized, Role: FOLLOWER
I20261120 03:01:46.493582  3534 consensus_queue.cc:261] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
I20261120 03:01:46.492363  3531 tablet_bootstrap.cc:654] T fb3606184a5141b8b0e80ba6902895db P c393ca4a78704704bc9b58916a81fc3f: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:46.498456  3529 log.cc:1199] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-2/wal/wals/9cc585adb4df48d18b69d8102382bf2e
I20261120 03:01:46.499289  3529 ts_tablet_manager.cc:1996] T 9cc585adb4df48d18b69d8102382bf2e P c393ca4a78704704bc9b58916a81fc3f: Deleting consensus metadata
I20261120 03:01:46.502530  3243 catalog_manager.cc:5081] TS c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993): tablet 9cc585adb4df48d18b69d8102382bf2e (table default.table_a938daa15a8b429190ccb7a61dab9f86 [id=957365c9d24645e985a308158ec0af66]) successfully deleted
I20261120 03:01:46.503506  3535 tablet_bootstrap.cc:492] T fb3606184a5141b8b0e80ba6902895db P d9ed1d903ef340ac8f1025c2b7df9bdd: No bootstrap required, opened a new log
I20261120 03:01:46.503901  3535 ts_tablet_manager.cc:1449] T fb3606184a5141b8b0e80ba6902895db P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent bootstrapping tablet: real 0.033s	user 0.007s	sys 0.006s
I20261120 03:01:46.506691  3535 raft_consensus.cc:366] T fb3606184a5141b8b0e80ba6902895db P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:46.507339  3535 raft_consensus.cc:392] T fb3606184a5141b8b0e80ba6902895db P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:46.507598  3535 raft_consensus.cc:787] T fb3606184a5141b8b0e80ba6902895db P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9ed1d903ef340ac8f1025c2b7df9bdd, State: Initialized, Role: FOLLOWER
I20261120 03:01:46.508291  3535 consensus_queue.cc:261] T fb3606184a5141b8b0e80ba6902895db P d9ed1d903ef340ac8f1025c2b7df9bdd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:46.510890  3530 ts_tablet_manager.cc:1975] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:46.511264  3530 log.cc:1199] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/9975a8096d0d4f9aa659a4d8af0ff0d9
I20261120 03:01:46.512125  3530 ts_tablet_manager.cc:1996] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting consensus metadata
    @     0x7f5aed506be2  kudu::master::HmsNotificationLogListenerTask::Init()::$_0::operator()() at ??:0
I20261120 03:01:46.522126  3534 ts_tablet_manager.cc:1480] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent starting tablet: real 0.034s	user 0.007s	sys 0.000s
I20261120 03:01:46.530833  3531 tablet_bootstrap.cc:492] T fb3606184a5141b8b0e80ba6902895db P c393ca4a78704704bc9b58916a81fc3f: No bootstrap required, opened a new log
I20261120 03:01:46.531301  3531 ts_tablet_manager.cc:1449] T fb3606184a5141b8b0e80ba6902895db P c393ca4a78704704bc9b58916a81fc3f: Time spent bootstrapping tablet: real 0.061s	user 0.011s	sys 0.000s
I20261120 03:01:46.534724  2910 heartbeater.cc:500] Master 127.0.96.126:46565 was elected leader, sending a full tablet report...
I20261120 03:01:46.536914  3532 ts_tablet_manager.cc:1975] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:46.537310  3532 log.cc:1199] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/9cc585adb4df48d18b69d8102382bf2e
I20261120 03:01:46.538141  3532 ts_tablet_manager.cc:1996] T 9cc585adb4df48d18b69d8102382bf2e P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting consensus metadata
I20261120 03:01:46.539167  3381 raft_consensus.cc:534] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:46.539593  3381 raft_consensus.cc:556] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
I20261120 03:01:46.541109  3243 catalog_manager.cc:5081] TS d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769): tablet 9cc585adb4df48d18b69d8102382bf2e (table default.table_a938daa15a8b429190ccb7a61dab9f86 [id=957365c9d24645e985a308158ec0af66]) successfully deleted
I20261120 03:01:46.541648  3381 leader_election.cc:290] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993), d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769)
I20261120 03:01:46.541826  3531 raft_consensus.cc:366] T fb3606184a5141b8b0e80ba6902895db P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:46.542595  3531 raft_consensus.cc:392] T fb3606184a5141b8b0e80ba6902895db P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:46.542886  3531 raft_consensus.cc:787] T fb3606184a5141b8b0e80ba6902895db P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c393ca4a78704704bc9b58916a81fc3f, State: Initialized, Role: FOLLOWER
I20261120 03:01:46.544171  3004 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "7b3a5f32068a44668cb454e6bc317f2a" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c393ca4a78704704bc9b58916a81fc3f" is_pre_election: true
I20261120 03:01:46.545562  2714 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "7b3a5f32068a44668cb454e6bc317f2a" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" is_pre_election: true
    @     0x7f5aed506b9a  _ZNSt3__18__invokeIRZN4kudu6master30HmsNotificationLogListenerTask4InitEvE3$_0JEEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOS6_DpOS7_ at ??:0
I20261120 03:01:46.546913  3052 heartbeater.cc:500] Master 127.0.96.126:46565 was elected leader, sending a full tablet report...
W20261120 03:01:46.549103  2799 leader_election.cc:343] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:46.549423  3535 ts_tablet_manager.cc:1480] T fb3606184a5141b8b0e80ba6902895db P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent starting tablet: real 0.045s	user 0.003s	sys 0.018s
W20261120 03:01:46.549750  2799 leader_election.cc:343] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769): Illegal state: must be running to vote when last-logged opid is not known
I20261120 03:01:46.549769  3531 consensus_queue.cc:261] T fb3606184a5141b8b0e80ba6902895db P c393ca4a78704704bc9b58916a81fc3f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:46.550133  2799 leader_election.cc:304] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 23741ccd042d4ca49afc3b8d1e26e22c; no voters: c393ca4a78704704bc9b58916a81fc3f, d9ed1d903ef340ac8f1025c2b7df9bdd
I20261120 03:01:46.552007  3381 raft_consensus.cc:2966] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20261120 03:01:46.555429  3249 catalog_manager.cc:5689] Ignoring report from unknown tablet 7c1a821b612b4325bc34a3a16c284b2d
I20261120 03:01:46.560268  3531 ts_tablet_manager.cc:1480] T fb3606184a5141b8b0e80ba6902895db P c393ca4a78704704bc9b58916a81fc3f: Time spent starting tablet: real 0.029s	user 0.007s	sys 0.000s
I20261120 03:01:46.560467  3535 tablet_bootstrap.cc:492] T 7b3a5f32068a44668cb454e6bc317f2a P d9ed1d903ef340ac8f1025c2b7df9bdd: Bootstrap starting.
I20261120 03:01:46.561620  3241 catalog_manager.cc:5081] TS 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): tablet 9975a8096d0d4f9aa659a4d8af0ff0d9 (table default.table_e387307c289c48c392b4a49829f76842 [id=2aefba03f3944dfc9f108a6431187ac3]) successfully deleted
I20261120 03:01:46.566334  3530 tablet_replica.cc:333] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c: stopping tablet replica
I20261120 03:01:46.567476  3532 tablet_replica.cc:333] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd: stopping tablet replica
I20261120 03:01:46.568198  3535 tablet_bootstrap.cc:654] T 7b3a5f32068a44668cb454e6bc317f2a P d9ed1d903ef340ac8f1025c2b7df9bdd: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:46.568475  3532 raft_consensus.cc:2440] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:46.568990  3532 raft_consensus.cc:2469] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd [term 1 FOLLOWER]: Raft consensus is shut down!
I20261120 03:01:46.571892  3530 raft_consensus.cc:2440] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 FOLLOWER]: Raft consensus shutting down.
I20261120 03:01:46.572407  3530 raft_consensus.cc:2469] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 FOLLOWER]: Raft consensus is shut down!
    @     0x7f5aed506b2a  std::__1::__invoke_void_return_wrapper<>::__call<>() at ??:0
I20261120 03:01:46.577365  2766 heartbeater.cc:500] Master 127.0.96.126:46565 was elected leader, sending a full tablet report...
I20261120 03:01:46.579700  3249 master_service.cc:965] ReplaceTablet: received request to replace tablet 9975a8096d0d4f9aa659a4d8af0ff0d9 from {username='slave'} at 127.0.0.1:35266
I20261120 03:01:46.584136  3249 master_service.cc:965] ReplaceTablet: received request to replace tablet 7c1a821b612b4325bc34a3a16c284b2d from {username='slave'} at 127.0.0.1:35266
I20261120 03:01:46.587984  3535 tablet_bootstrap.cc:492] T 7b3a5f32068a44668cb454e6bc317f2a P d9ed1d903ef340ac8f1025c2b7df9bdd: No bootstrap required, opened a new log
I20261120 03:01:46.588429  3535 ts_tablet_manager.cc:1449] T 7b3a5f32068a44668cb454e6bc317f2a P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent bootstrapping tablet: real 0.028s	user 0.010s	sys 0.000s
I20261120 03:01:46.589989  3531 tablet_bootstrap.cc:492] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f: Bootstrap starting.
I20261120 03:01:46.591387  3535 raft_consensus.cc:366] T 7b3a5f32068a44668cb454e6bc317f2a P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
I20261120 03:01:46.592077  3535 raft_consensus.cc:392] T 7b3a5f32068a44668cb454e6bc317f2a P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:46.593465  3530 ts_tablet_manager.cc:1962] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:46.592387  3535 raft_consensus.cc:787] T 7b3a5f32068a44668cb454e6bc317f2a P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9ed1d903ef340ac8f1025c2b7df9bdd, State: Initialized, Role: FOLLOWER
I20261120 03:01:46.600495  3535 consensus_queue.cc:261] T 7b3a5f32068a44668cb454e6bc317f2a P d9ed1d903ef340ac8f1025c2b7df9bdd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
    @     0x7f5aed506af2  std::__1::__function::__alloc_func<>::operator()() at ??:0
I20261120 03:01:46.604130  3534 tablet_bootstrap.cc:492] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c: Bootstrap starting.
I20261120 03:01:46.602802  3535 ts_tablet_manager.cc:1480] T 7b3a5f32068a44668cb454e6bc317f2a P d9ed1d903ef340ac8f1025c2b7df9bdd: Time spent starting tablet: real 0.014s	user 0.008s	sys 0.000s
W20261120 03:01:46.623109  3249 catalog_manager.cc:5689] Ignoring report from unknown tablet 7c1a821b612b4325bc34a3a16c284b2d
    @     0x7f5aed505dee  std::__1::__function::__func<>::operator()() at ??:0
    @           0x327655  std::__1::__function::__value_func<>::operator()() at /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/../sanitizer_common/sanitizer_vector.h:111
    @           0x32747a  std::__1::function<>::operator()() at /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp:294
I20261120 03:01:46.629664  3534 tablet_bootstrap.cc:654] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c: Neither blocks nor log segments found. Creating new log.
I20261120 03:01:46.632054  3532 ts_tablet_manager.cc:1962] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting tablet data with delete state TABLET_DATA_DELETED
I20261120 03:01:46.642100  3531 tablet_bootstrap.cc:654] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f: Neither blocks nor log segments found. Creating new log.
    @     0x7f5ae45157db  kudu::Thread::SuperviseThread() at ??:0
    @           0x277de0  __tsan_thread_start_func at ??:0
I20261120 03:01:46.656077  3534 tablet_bootstrap.cc:492] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c: No bootstrap required, opened a new log
I20261120 03:01:46.657312  3534 ts_tablet_manager.cc:1449] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent bootstrapping tablet: real 0.053s	user 0.017s	sys 0.000s
I20261120 03:01:46.659963  3534 raft_consensus.cc:366] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:46.660602  3534 raft_consensus.cc:392] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:46.663546  3534 raft_consensus.cc:787] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23741ccd042d4ca49afc3b8d1e26e22c, State: Initialized, Role: FOLLOWER
I20261120 03:01:46.664402  3534 consensus_queue.cc:261] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:46.665088  3532 ts_tablet_manager.cc:1975] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:46.672598  3534 ts_tablet_manager.cc:1480] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c: Time spent starting tablet: real 0.015s	user 0.005s	sys 0.000s
    @     0x7f5ae5c836db  start_thread at ??:0
I20261120 03:01:46.677340  3531 tablet_bootstrap.cc:492] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f: No bootstrap required, opened a new log
I20261120 03:01:46.677780  3531 ts_tablet_manager.cc:1449] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f: Time spent bootstrapping tablet: real 0.088s	user 0.007s	sys 0.003s
I20261120 03:01:46.686758  3530 ts_tablet_manager.cc:1975] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c: tablet deleted with delete type TABLET_DATA_DELETED: last-logged OpId 1.1
I20261120 03:01:46.665453  3532 log.cc:1199] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-0/wal/wals/9975a8096d0d4f9aa659a4d8af0ff0d9
I20261120 03:01:46.688221  3532 ts_tablet_manager.cc:1996] T 9975a8096d0d4f9aa659a4d8af0ff0d9 P d9ed1d903ef340ac8f1025c2b7df9bdd: Deleting consensus metadata
I20261120 03:01:46.689615  3530 log.cc:1199] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting WAL directory at /tmp/dist-test-tasklxEiy3/test-tmp/master-stress-test.0.CatalogManagerConfigurations_MasterStressTest.Test_3.1795143632597315-385-0/minicluster-data/ts-1/wal/wals/9cc585adb4df48d18b69d8102382bf2e
I20261120 03:01:46.690359  3530 ts_tablet_manager.cc:1996] T 9cc585adb4df48d18b69d8102382bf2e P 23741ccd042d4ca49afc3b8d1e26e22c: Deleting consensus metadata
    @     0x7f5ae0fb961f  clone at ??:0
I20261120 03:01:46.690807  3531 raft_consensus.cc:366] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
I20261120 03:01:46.694506  3531 raft_consensus.cc:392] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20261120 03:01:46.694808  3531 raft_consensus.cc:787] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c393ca4a78704704bc9b58916a81fc3f, State: Initialized, Role: FOLLOWER
I20261120 03:01:46.695511  3531 consensus_queue.cc:261] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
I20261120 03:01:46.708154  3531 ts_tablet_manager.cc:1480] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f: Time spent starting tablet: real 0.030s	user 0.005s	sys 0.003s
I20261120 03:01:46.725307  3546 raft_consensus.cc:534] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:46.725786  3546 raft_consensus.cc:556] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:46.727842  3546 leader_election.cc:290] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769), c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993)
I20261120 03:01:46.730391  2714 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "fb3606184a5141b8b0e80ba6902895db" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" is_pre_election: true
I20261120 03:01:46.731106  2714 raft_consensus.cc:2062] T fb3606184a5141b8b0e80ba6902895db P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 23741ccd042d4ca49afc3b8d1e26e22c in term 0
I20261120 03:01:46.731997  3004 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "fb3606184a5141b8b0e80ba6902895db" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c393ca4a78704704bc9b58916a81fc3f" is_pre_election: true
I20261120 03:01:46.732681  3004 raft_consensus.cc:2062] T fb3606184a5141b8b0e80ba6902895db P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate 23741ccd042d4ca49afc3b8d1e26e22c in term 0
I20261120 03:01:46.734342  2799 leader_election.cc:304] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 23741ccd042d4ca49afc3b8d1e26e22c, d9ed1d903ef340ac8f1025c2b7df9bdd; no voters: 
I20261120 03:01:46.736581  3546 raft_consensus.cc:3024] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:46.736984  3546 raft_consensus.cc:534] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:46.737345  3546 raft_consensus.cc:3296] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:46.744472  3546 raft_consensus.cc:556] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:46.748082  2714 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "fb3606184a5141b8b0e80ba6902895db" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd"
I20261120 03:01:46.748602  2714 raft_consensus.cc:3296] T fb3606184a5141b8b0e80ba6902895db P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:46.749467  2714 raft_consensus.cc:2062] T fb3606184a5141b8b0e80ba6902895db P d9ed1d903ef340ac8f1025c2b7df9bdd [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 23741ccd042d4ca49afc3b8d1e26e22c in term 1
I20261120 03:01:46.753744  3004 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "fb3606184a5141b8b0e80ba6902895db" candidate_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c393ca4a78704704bc9b58916a81fc3f"
I20261120 03:01:46.754338  3004 raft_consensus.cc:3296] T fb3606184a5141b8b0e80ba6902895db P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:46.762698  3004 raft_consensus.cc:2062] T fb3606184a5141b8b0e80ba6902895db P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate 23741ccd042d4ca49afc3b8d1e26e22c in term 1
I20261120 03:01:46.764251  2799 leader_election.cc:304] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 23741ccd042d4ca49afc3b8d1e26e22c, d9ed1d903ef340ac8f1025c2b7df9bdd; no voters: 
I20261120 03:01:46.766579  3381 raft_consensus.cc:3024] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:46.767361  3546 leader_election.cc:290] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [CANDIDATE]: Term 1 election: Requested vote from peers d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769), c393ca4a78704704bc9b58916a81fc3f (127.0.96.67:40993)
I20261120 03:01:46.781458  3381 raft_consensus.cc:744] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 LEADER]: Becoming Leader. State: Replica: 23741ccd042d4ca49afc3b8d1e26e22c, State: Running, Role: LEADER
I20261120 03:01:46.789407  3381 consensus_queue.cc:238] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } }
I20261120 03:01:46.825379  3546 consensus_queue.cc:1049] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:46.839279  3546 consensus_queue.cc:1049] T fb3606184a5141b8b0e80ba6902895db P 23741ccd042d4ca49afc3b8d1e26e22c [LEADER]: Connected to new peer: Peer: permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:46.855811  3556 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:46.864583  3556 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:46.872386  3556 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:01:46.913021  3431 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:01:46.942999  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:46.957834  3541 raft_consensus.cc:534] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20261120 03:01:46.958364  3541 raft_consensus.cc:556] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
I20261120 03:01:46.960394  3541 leader_election.cc:290] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733), d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769)
I20261120 03:01:46.962867  2862 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "7b3a5f32068a44668cb454e6bc317f2a" candidate_uuid: "c393ca4a78704704bc9b58916a81fc3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" is_pre_election: true
I20261120 03:01:46.963631  2862 raft_consensus.cc:2062] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Leader pre-election vote request: granting yes vote for candidate c393ca4a78704704bc9b58916a81fc3f in term 0
I20261120 03:01:46.965128  2939 leader_election.cc:304] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 23741ccd042d4ca49afc3b8d1e26e22c, c393ca4a78704704bc9b58916a81fc3f; no voters: 
I20261120 03:01:46.966109  3541 raft_consensus.cc:3024] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20261120 03:01:46.966605  3541 raft_consensus.cc:534] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20261120 03:01:46.966995  3541 raft_consensus.cc:3296] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:46.974260  3541 raft_consensus.cc:556] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
I20261120 03:01:46.975557  2714 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "7b3a5f32068a44668cb454e6bc317f2a" candidate_uuid: "c393ca4a78704704bc9b58916a81fc3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" is_pre_election: true
I20261120 03:01:46.978323  2862 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "7b3a5f32068a44668cb454e6bc317f2a" candidate_uuid: "c393ca4a78704704bc9b58916a81fc3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "23741ccd042d4ca49afc3b8d1e26e22c"
I20261120 03:01:46.978874  2862 raft_consensus.cc:3296] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:46.980021  2862 raft_consensus.cc:2062] T 7b3a5f32068a44668cb454e6bc317f2a P 23741ccd042d4ca49afc3b8d1e26e22c [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate c393ca4a78704704bc9b58916a81fc3f in term 1
I20261120 03:01:46.982136  2715 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "7b3a5f32068a44668cb454e6bc317f2a" candidate_uuid: "c393ca4a78704704bc9b58916a81fc3f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd"
I20261120 03:01:46.982720  2715 raft_consensus.cc:3296] T 7b3a5f32068a44668cb454e6bc317f2a P d9ed1d903ef340ac8f1025c2b7df9bdd [term 0 FOLLOWER]: Advancing to term 1
I20261120 03:01:46.983615  2715 raft_consensus.cc:2062] T 7b3a5f32068a44668cb454e6bc317f2a P d9ed1d903ef340ac8f1025c2b7df9bdd [term 1 FOLLOWER]: Leader election vote request: granting yes vote for candidate c393ca4a78704704bc9b58916a81fc3f in term 1
I20261120 03:01:46.991163  2941 leader_election.cc:304] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c393ca4a78704704bc9b58916a81fc3f, d9ed1d903ef340ac8f1025c2b7df9bdd; no voters: 
I20261120 03:01:46.993027  3518 raft_consensus.cc:3024] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [term 1 FOLLOWER]: Leader election won for term 1
I20261120 03:01:46.993557  2714 raft_consensus.cc:2612] T 7b3a5f32068a44668cb454e6bc317f2a P d9ed1d903ef340ac8f1025c2b7df9bdd [term 1 FOLLOWER]: Leader pre-election vote request: already granted 'yes' vote to c393ca4a78704704bc9b58916a81fc3f in term 1; re-sending same reply
I20261120 03:01:46.994011  3541 leader_election.cc:290] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [CANDIDATE]: Term 1 election: Requested vote from peers 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733), d9ed1d903ef340ac8f1025c2b7df9bdd (127.0.96.65:32769)
I20261120 03:01:47.007418  3518 raft_consensus.cc:744] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [term 1 LEADER]: Becoming Leader. State: Replica: c393ca4a78704704bc9b58916a81fc3f, State: Running, Role: LEADER
I20261120 03:01:47.008409  3518 consensus_queue.cc:238] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 } } peers { permanent_uuid: "c393ca4a78704704bc9b58916a81fc3f" member_type: VOTER last_known_addr { host: "127.0.96.67" port: 40993 } } peers { permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 } }
I20261120 03:01:47.017848  3564 consensus_queue.cc:1049] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [LEADER]: Connected to new peer: Peer: permanent_uuid: "23741ccd042d4ca49afc3b8d1e26e22c" member_type: VOTER last_known_addr { host: "127.0.96.66" port: 43733 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20261120 03:01:47.019589  3541 consensus_queue.cc:1049] T 7b3a5f32068a44668cb454e6bc317f2a P c393ca4a78704704bc9b58916a81fc3f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d9ed1d903ef340ac8f1025c2b7df9bdd" member_type: VOTER last_known_addr { host: "127.0.96.65" port: 32769 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20261120 03:01:47.063314  2651 connection.cc:573] server connection from 127.0.0.1:38248 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:47.064282  2797 connection.cc:573] server connection from 127.0.0.1:43676 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:47.065220  3433 connection.cc:573] client connection to 127.0.96.126:46565 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:47.065382  2940 connection.cc:573] client connection to 127.0.96.126:46565 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:47.067330  2650 connection.cc:573] client connection to 127.0.96.126:46565 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:47.068164  3062 connection.cc:573] client connection to 127.0.96.126:46565 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
W20261120 03:01:47.069885  3062 master_proxy_rpc.cc:203] Re-attempting ReplaceTablet request to leader Master (127.0.96.126:46565)
W20261120 03:01:47.070567  3062 master_proxy_rpc.cc:203] Re-attempting CreateTable request to leader Master (127.0.96.126:46565)
W20261120 03:01:47.071532  3062 master_proxy_rpc.cc:203] Re-attempting DeleteTable request to leader Master (127.0.96.126:46565)
W20261120 03:01:47.076084  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:01:47.076514  3062 master_proxy_rpc.cc:203] Re-attempting AlterTable request to leader Master (127.0.96.126:46565)
I20261120 03:01:47.076540  3433 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:01:47.079521  3556 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:01:47.784365  3576 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:47.784837  3576 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:47.786844  3576 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:01:47.826148  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:01:47.830155  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:47.830653  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:01:47.831468  3576 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:01:48.234727   385 master-stress-test.cc:579] Killing test thread 1/10
I20261120 03:01:48.666484  3583 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:48.667740  3583 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:48.670470  3583 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:01:48.674391  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:01:48.674815  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:48.675119  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:01:48.675647  3583 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
W20261120 03:01:48.741669  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 6: this message will repeat every 5th retry.
I20261120 03:01:49.795251  3598 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:49.795647  3598 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:49.797175  3598 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:01:49.801870  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:01:49.802353  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:49.802721  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:01:49.803395  3598 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:01:50.680568  3609 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:50.680958  3609 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:50.682440  3609 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:01:50.687449  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:01:50.687873  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:50.688222  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:01:50.688735  3609 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:01:50.993052  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:01:50.993436  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:01:51.500926  3618 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:51.501382  3618 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:51.503001  3618 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:01:51.507722  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:01:51.508149  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:51.508474  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:01:51.509017  3618 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:01:53.486294  3634 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:53.486708  3634 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:53.488184  3634 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:01:53.490170  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 11 similar messages]
W20261120 03:01:53.492946  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:01:53.493325  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
I20261120 03:01:53.493629  3433 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:01:53.494213  3634 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:01:55.996832  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:01:55.997277  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:01:58.169042  3693 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:01:58.169720  3693 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:01:58.171506  3693 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:01:58.176620  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:01:58.177081  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:01:58.177395  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:01:58.178030  3693 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
W20261120 03:01:58.790263  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 11: this message will repeat every 5th retry.
I20261120 03:02:01.000232  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:02:01.000602  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:02:05.786861  3777 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:02:05.787395  3777 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:02:05.789140  3777 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:02:05.791885  3431 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111) [suppressed 3 similar messages]
W20261120 03:02:05.794588  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:02:05.794998  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:02:05.795296  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:02:05.795871  3777 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:02:06.002985  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:02:06.003446  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:02:08.117465  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 16: this message will repeat every 5th retry.
I20261120 03:02:11.006274  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:02:11.006619  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:02:16.010089  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:02:16.010519  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:02:18.213946  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 21: this message will repeat every 5th retry.
I20261120 03:02:21.013639  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:02:21.014017  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:02:22.863756  3917 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:02:22.864248  3917 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:02:22.866230  3917 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:02:22.868366  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20261120 03:02:22.871363  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:02:22.871785  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
I20261120 03:02:22.872081  3433 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:02:22.872661  3917 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:02:26.017622  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:02:26.018139  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:02:27.671878  3966 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:02:27.672372  3966 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:02:27.673993  3966 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:02:27.678900  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:02:27.679522  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:02:27.679867  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:02:27.680464  3966 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
W20261120 03:02:28.134099  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 26: this message will repeat every 5th retry.
W20261120 03:02:29.291723  3478 debug-util.cc:398] Leaking SignalData structure 0x7b08000b39c0 after lost signal to thread 3405
W20261120 03:02:29.292299  3478 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8cc0 after lost signal to thread 3481
I20261120 03:02:31.036827  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:02:31.037212  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:02:36.040688  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:02:36.041147  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:02:36.266001  2760 heartbeater.cc:648] Failed to heartbeat to 127.0.96.124:33751 (124 consecutive failures): Network error: Failed to ping master at 127.0.96.124:33751: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:02:36.367197  2910 heartbeater.cc:648] Failed to heartbeat to 127.0.96.126:46565 (100 consecutive failures): Network error: Failed to ping master at 127.0.96.126:46565: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:02:36.778995  3050 heartbeater.cc:648] Failed to heartbeat to 127.0.96.124:33751 (162 consecutive failures): Network error: Failed to ping master at 127.0.96.124:33751: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:02:38.109246  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 31: this message will repeat every 5th retry.
I20261120 03:02:41.043964  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:02:41.044328  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:02:45.482280  4114 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:02:45.482908  4114 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:02:45.485224  4114 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:02:45.488214  3431 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111) [suppressed 3 similar messages]
W20261120 03:02:45.490862  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:02:45.491240  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:02:45.491551  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:02:45.492154  4114 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:02:46.047199  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:02:46.047639  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:02:47.665834  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 36: this message will repeat every 5th retry.
I20261120 03:02:51.050269  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:02:51.050637  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:02:51.221930  4179 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:02:51.222376  4179 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:02:51.224092  4179 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:02:51.225598  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20261120 03:02:51.229058  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:02:51.229533  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:02:51.229858  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:02:51.230441  4179 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:02:53.368697  4205 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:02:53.369102  4205 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:02:53.370833  4205 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:02:53.375351  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:02:53.375771  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:02:53.376085  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:02:53.376624  4205 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:02:54.822832  4229 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:02:54.823429  4229 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:02:54.825328  4229 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:02:54.831301  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:02:54.831871  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:02:54.832201  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:02:54.832849  4229 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:02:56.053737  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:02:56.054209  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:02:57.745229  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 41: this message will repeat every 5th retry.
I20261120 03:03:00.440378  4296 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:03:00.440783  4296 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:03:00.442435  4296 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:03:00.444478  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 5 similar messages]
W20261120 03:03:00.447439  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:03:00.447892  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
I20261120 03:03:00.448215  3433 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:03:00.448769  4296 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:03:01.057024  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:03:01.057406  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:03:06.061007  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:03:06.061494  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:03:07.853058  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 46: this message will repeat every 5th retry.
W20261120 03:03:08.273350  3046 debug-util.cc:398] Leaking SignalData structure 0x7b08000dd220 after lost signal to thread 2920
W20261120 03:03:08.274350  3046 debug-util.cc:398] Leaking SignalData structure 0x7b08000d7880 after lost signal to thread 3049
W20261120 03:03:08.512749  3478 debug-util.cc:398] Leaking SignalData structure 0x7b08000b2280 after lost signal to thread 3405
W20261120 03:03:08.513289  3478 debug-util.cc:398] Leaking SignalData structure 0x7b0800050a80 after lost signal to thread 3481
W20261120 03:03:10.663738  2756 debug-util.cc:398] Leaking SignalData structure 0x7b0800042ac0 after lost signal to thread 2624
W20261120 03:03:10.664745  2756 debug-util.cc:398] Leaking SignalData structure 0x7b08000cb6e0 after lost signal to thread 2759
I20261120 03:03:11.098137  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:03:11.098531  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:03:12.682590  4430 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:03:12.683113  4430 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:03:12.684779  4430 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:03:12.687075  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20261120 03:03:12.689982  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:03:12.690455  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
I20261120 03:03:12.690783  3433 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:03:12.691375  4430 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:03:16.102116  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:03:16.102560  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:03:17.613042  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 51: this message will repeat every 5th retry.
I20261120 03:03:21.105254  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:03:21.105587  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:03:26.109223  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:03:26.109655  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:03:27.239873  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 56: this message will repeat every 5th retry.
I20261120 03:03:27.840595  4602 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:03:27.841084  4602 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:03:27.843129  4602 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:03:27.846767  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20261120 03:03:27.850358  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:03:27.850947  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:03:27.851294  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:03:27.851907  4602 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:03:31.112435  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:03:31.112824  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:03:33.014703  2904 debug-util.cc:398] Leaking SignalData structure 0x7b08000cd7e0 after lost signal to thread 2772
W20261120 03:03:33.015949  2904 debug-util.cc:398] Leaking SignalData structure 0x7b0800042f80 after lost signal to thread 2907
I20261120 03:03:33.204237  4672 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:03:33.204617  4672 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:03:33.206355  4672 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:03:33.208220  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20261120 03:03:33.211632  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:03:33.212929  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:03:33.213356  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:03:33.214038  4672 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:03:36.116775  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:03:36.117208  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:03:36.576558  2766 heartbeater.cc:648] Failed to heartbeat to 127.0.96.126:46565 (219 consecutive failures): Network error: Failed to ping master at 127.0.96.126:46565: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:03:36.757790  2910 heartbeater.cc:648] Failed to heartbeat to 127.0.96.126:46565 (220 consecutive failures): Network error: Failed to ping master at 127.0.96.126:46565: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:03:37.120429  3052 heartbeater.cc:648] Failed to heartbeat to 127.0.96.126:46565 (279 consecutive failures): Network error: Failed to ping master at 127.0.96.126:46565: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:03:37.480887  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 61: this message will repeat every 5th retry.
I20261120 03:03:41.120153  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:03:41.120507  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:03:45.926236  3436 maintenance_manager.cc:419] P a23f052846654c9daf3fb0d90fbb6f09: Scheduling FlushMRSOp(00000000000000000000000000000000): perf score=0.033346
I20261120 03:03:46.043133  3435 maintenance_manager.cc:643] P a23f052846654c9daf3fb0d90fbb6f09: FlushMRSOp(00000000000000000000000000000000) complete. Timing: real 0.116s	user 0.043s	sys 0.019s Metrics: {"bytes_written":11010,"cfile_init":1,"drs_written":1,"lbm_read_time_us":203,"lbm_reads_lt_1ms":4,"lbm_write_time_us":12877,"lbm_writes_10-100_ms":1,"lbm_writes_lt_1ms":29,"peak_mem_usage":0,"rows_written":18,"thread_start_us":571,"threads_started":1,"wal-append.queue_time_us":904}
I20261120 03:03:46.045153  3436 maintenance_manager.cc:419] P a23f052846654c9daf3fb0d90fbb6f09: Scheduling UndoDeltaBlockGCOp(00000000000000000000000000000000): 1146 bytes on disk
I20261120 03:03:46.046675  3435 maintenance_manager.cc:643] P a23f052846654c9daf3fb0d90fbb6f09: UndoDeltaBlockGCOp(00000000000000000000000000000000) complete. Timing: real 0.001s	user 0.001s	sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":215,"lbm_reads_lt_1ms":4}
I20261120 03:03:46.123700  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:03:46.124114  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:03:46.973208  2904 debug-util.cc:398] Leaking SignalData structure 0x7b08000421c0 after lost signal to thread 2772
W20261120 03:03:46.974403  2904 debug-util.cc:398] Leaking SignalData structure 0x7b08000c2280 after lost signal to thread 2907
W20261120 03:03:47.656591  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 66: this message will repeat every 5th retry.
I20261120 03:03:51.126682  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:03:51.127043  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:03:53.184549  4976 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:03:53.184990  4976 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:03:53.186847  4976 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:03:53.188521  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20261120 03:03:53.191308  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:03:53.191913  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:03:53.192193  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:03:53.192734  4976 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
W20261120 03:03:55.305514  2756 debug-util.cc:398] Leaking SignalData structure 0x7b08000b3e80 after lost signal to thread 2624
W20261120 03:03:55.306468  2756 debug-util.cc:398] Leaking SignalData structure 0x7b08000db520 after lost signal to thread 2759
W20261120 03:03:55.875640  2904 debug-util.cc:398] Leaking SignalData structure 0x7b08000d8740 after lost signal to thread 2772
W20261120 03:03:55.876516  2904 debug-util.cc:398] Leaking SignalData structure 0x7b08000d2a40 after lost signal to thread 2907
I20261120 03:03:56.130640  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:03:56.131073  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:03:56.753937  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 71: this message will repeat every 5th retry.
I20261120 03:04:01.133700  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:04:01.134146  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:04:06.137549  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:04:06.137988  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:04:07.348351  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 76: this message will repeat every 5th retry.
W20261120 03:04:08.803195  2756 debug-util.cc:398] Leaking SignalData structure 0x7b08000d0f00 after lost signal to thread 2624
W20261120 03:04:08.804024  2756 debug-util.cc:398] Leaking SignalData structure 0x7b08000db680 after lost signal to thread 2759
I20261120 03:04:11.140861  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:04:11.141233  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:04:11.383891  5308 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:04:11.384383  5308 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:04:11.386464  5308 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:04:11.387660  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20261120 03:04:11.390718  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:04:11.391203  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:04:11.391472  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:04:11.392019  5308 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:04:12.048000  5323 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:04:12.048406  5323 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:04:12.049962  5323 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:04:12.054281  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:04:12.054648  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:04:12.054913  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:04:12.055362  5323 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:04:16.144603  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:04:16.145035  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:04:16.872471  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 81: this message will repeat every 5th retry.
I20261120 03:04:21.147850  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:04:21.148190  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:04:26.151641  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:04:26.152079  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:04:27.153733  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 86: this message will repeat every 5th retry.
I20261120 03:04:28.261227  5570 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:04:28.261732  5570 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:04:28.263741  5570 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:04:28.265782  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 3 similar messages]
W20261120 03:04:28.268709  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:04:28.269191  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:04:28.269450  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:04:28.269996  5570 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:04:31.154930  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:04:31.155285  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:04:33.658587  5672 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:04:33.658963  5672 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:04:33.660545  5672 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:04:33.662240  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20261120 03:04:33.665159  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:04:33.665521  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:04:33.665845  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:04:33.666332  5672 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:04:36.158864  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:04:36.159301  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:04:36.565132  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 91: this message will repeat every 5th retry.
W20261120 03:04:36.950498  2766 heartbeater.cc:648] Failed to heartbeat to 127.0.96.126:46565 (339 consecutive failures): Network error: Failed to ping master at 127.0.96.126:46565: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:04:37.142549  2910 heartbeater.cc:648] Failed to heartbeat to 127.0.96.126:46565 (340 consecutive failures): Network error: Failed to ping master at 127.0.96.126:46565: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:04:37.574970  3050 heartbeater.cc:648] Failed to heartbeat to 127.0.96.124:33751 (460 consecutive failures): Network error: Failed to ping master at 127.0.96.124:33751: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:04:38.736122  5769 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:04:38.736598  5769 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:04:38.738299  5769 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:04:38.740397  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20261120 03:04:38.743085  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:04:38.743472  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
I20261120 03:04:38.743746  3433 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:04:38.744282  5769 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
W20261120 03:04:39.834960  2756 debug-util.cc:398] Leaking SignalData structure 0x7b08000b35c0 after lost signal to thread 2624
W20261120 03:04:39.836576  2756 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4120 after lost signal to thread 2759
W20261120 03:04:40.378502  3046 debug-util.cc:398] Leaking SignalData structure 0x7b08000c7220 after lost signal to thread 2920
W20261120 03:04:40.379325  3046 debug-util.cc:398] Leaking SignalData structure 0x7b08000c2020 after lost signal to thread 3049
I20261120 03:04:41.161949  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:04:41.162348  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:04:41.192930  5816 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:04:41.193394  5816 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:04:41.195026  5816 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:04:41.198697  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:04:41.199128  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:04:41.199404  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:04:41.199885  5816 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:04:46.165704  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:04:46.166155  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:04:46.302362  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 96: this message will repeat every 5th retry.
I20261120 03:04:51.168941  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:04:51.169355  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:04:52.847918  6021 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:04:52.848502  6021 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:04:52.850742  6021 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:04:52.852970  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 3 similar messages]
W20261120 03:04:52.856174  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:04:52.856693  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:04:52.856979  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:04:52.857573  6021 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
W20261120 03:04:55.946643  3478 debug-util.cc:398] Leaking SignalData structure 0x7b08000ab460 after lost signal to thread 3405
W20261120 03:04:55.947259  3478 debug-util.cc:398] Leaking SignalData structure 0x7b080004fee0 after lost signal to thread 3481
W20261120 03:04:56.065495  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 101: this message will repeat every 5th retry.
I20261120 03:04:56.195088  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:04:56.195508  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:05:01.198417  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:05:01.198807  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:05:05.759678  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 106: this message will repeat every 5th retry.
I20261120 03:05:06.202188  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:05:06.202627  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:05:07.508044  6256 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:05:07.508519  6256 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:05:07.510180  6256 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:05:07.512168  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20261120 03:05:07.514724  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:05:07.515239  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:05:07.515507  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:05:07.516040  6256 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:05:08.924189  6283 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:05:08.924573  6283 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:05:08.926399  6283 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:05:08.931967  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:05:08.932862  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:05:08.933137  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:05:08.933738  6283 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:05:11.205201  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:05:11.205567  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:05:14.962568  3478 debug-util.cc:398] Leaking SignalData structure 0x7b08000b24a0 after lost signal to thread 3405
W20261120 03:05:14.963367  3478 debug-util.cc:398] Leaking SignalData structure 0x7b080009a060 after lost signal to thread 3481
I20261120 03:05:16.241925  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:05:16.242348  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:05:16.393550  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 111: this message will repeat every 5th retry.
I20261120 03:05:21.244824  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:05:21.245211  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:05:26.249243  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:05:26.249662  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:05:26.281888  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 116: this message will repeat every 5th retry.
I20261120 03:05:27.742628  6602 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:05:27.743108  6602 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:05:27.745080  6602 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:05:27.747478  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 3 similar messages]
W20261120 03:05:27.750669  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:05:27.751025  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
I20261120 03:05:27.751341  3433 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:05:27.751869  6602 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:05:31.252264  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:05:31.252607  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:05:36.256150  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:05:36.256613  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:05:36.860998  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 121: this message will repeat every 5th retry.
W20261120 03:05:37.333572  2760 heartbeater.cc:648] Failed to heartbeat to 127.0.96.124:33751 (484 consecutive failures): Network error: Failed to ping master at 127.0.96.124:33751: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:05:37.510780  2910 heartbeater.cc:648] Failed to heartbeat to 127.0.96.126:46565 (460 consecutive failures): Network error: Failed to ping master at 127.0.96.126:46565: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:05:37.866287  3050 heartbeater.cc:648] Failed to heartbeat to 127.0.96.124:33751 (606 consecutive failures): Network error: Failed to ping master at 127.0.96.124:33751: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:05:38.293742  2904 debug-util.cc:398] Leaking SignalData structure 0x7b08000dbba0 after lost signal to thread 2772
W20261120 03:05:38.294875  2904 debug-util.cc:398] Leaking SignalData structure 0x7b08000c26a0 after lost signal to thread 2907
I20261120 03:05:39.393391  6808 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:05:39.393972  6808 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:05:39.396036  6808 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:05:39.415048  3431 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20261120 03:05:39.419632  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:05:39.421345  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
I20261120 03:05:39.421705  3433 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:05:39.422442  6808 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:05:41.259640  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:05:41.259989  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:05:43.202361  6880 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:05:43.202749  6880 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:05:43.204277  6880 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:05:43.209738  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:05:43.210181  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:05:43.210477  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:05:43.211004  6880 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:05:46.263275  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:05:46.263686  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:05:46.815557  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 126: this message will repeat every 5th retry.
I20261120 03:05:51.266557  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:05:51.266959  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:05:56.270494  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:05:56.270965  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:05:56.437546  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 131: this message will repeat every 5th retry.
I20261120 03:06:01.273852  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:06:01.274257  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:06:01.326045  7205 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:06:01.326581  7205 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:06:01.329164  7205 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:06:01.331730  3431 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111) [suppressed 3 similar messages]
W20261120 03:06:01.335930  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:06:01.336494  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
I20261120 03:06:01.336869  3433 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:06:01.337495  7205 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:06:04.321228  7261 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:06:04.321645  7261 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:06:04.323225  7261 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:06:04.327306  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:06:04.327641  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:06:04.327940  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:06:04.328428  7261 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:06:06.277571  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:06:06.277984  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:06:06.801949  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 136: this message will repeat every 5th retry.
I20261120 03:06:11.280757  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:06:11.281114  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:06:16.284480  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:06:16.284893  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:06:16.626549  7483 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:06:16.627063  7483 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:06:16.628718  7483 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:06:16.630029  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 3 similar messages]
W20261120 03:06:16.632853  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:06:16.633334  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:06:16.633589  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:06:16.634111  7483 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
W20261120 03:06:16.780218  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 141: this message will repeat every 5th retry.
I20261120 03:06:21.287482  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:06:21.287824  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:06:26.291208  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:06:26.291631  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:06:26.517082  2756 debug-util.cc:398] Leaking SignalData structure 0x7b08000d1fa0 after lost signal to thread 2624
W20261120 03:06:26.518021  2756 debug-util.cc:398] Leaking SignalData structure 0x7b08000c2320 after lost signal to thread 2759
W20261120 03:06:26.708214  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 146: this message will repeat every 5th retry.
I20261120 03:06:31.294265  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:06:31.294616  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:06:32.477602  3046 debug-util.cc:398] Leaking SignalData structure 0x7b08000e3360 after lost signal to thread 2920
W20261120 03:06:32.478436  3046 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba020 after lost signal to thread 3049
I20261120 03:06:34.098373  7774 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:06:34.098870  7774 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:06:34.100832  7774 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:06:34.102845  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20261120 03:06:34.105661  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:06:34.106055  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
I20261120 03:06:34.106387  3433 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:06:34.106921  7774 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:06:36.297984  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:06:36.298449  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
W20261120 03:06:36.312842  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 151: this message will repeat every 5th retry.
W20261120 03:06:37.721344  2766 heartbeater.cc:648] Failed to heartbeat to 127.0.96.126:46565 (579 consecutive failures): Network error: Failed to ping master at 127.0.96.126:46565: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:06:37.893705  3052 heartbeater.cc:648] Failed to heartbeat to 127.0.96.126:46565 (724 consecutive failures): Network error: Failed to ping master at 127.0.96.126:46565: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:06:37.898674  2908 heartbeater.cc:648] Failed to heartbeat to 127.0.96.124:33751 (606 consecutive failures): Network error: Failed to ping master at 127.0.96.124:33751: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:06:41.301159  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:06:41.301496  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:06:44.309417  7944 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:06:44.309895  7944 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:06:44.311475  7944 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:06:44.313683  3433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111) [suppressed 1 similar messages]
W20261120 03:06:44.316429  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
W20261120 03:06:44.316787  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
I20261120 03:06:44.317066  3433 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:06:44.317732  7944 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:06:44.905915  7956 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:06:44.906515  7956 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:06:44.908246  7956 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:06:44.913506  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:06:44.913954  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:06:44.914268  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:06:44.914808  7956 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:06:46.304083  3400 master_runner.cc:187] Error getting master registration for 127.0.96.126:46565: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111), 
I20261120 03:06:46.304479  3400 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
I20261120 03:06:46.397070  7978 raft_consensus.cc:534] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader 6e18f436ef9945d78f543c3471320ee7)
I20261120 03:06:46.397482  7978 raft_consensus.cc:556] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e18f436ef9945d78f543c3471320ee7" member_type: VOTER last_known_addr { host: "127.0.96.126" port: 46565 } } peers { permanent_uuid: "a23f052846654c9daf3fb0d90fbb6f09" member_type: VOTER last_known_addr { host: "127.0.96.125" port: 36419 } } peers { permanent_uuid: "86ed298871604dbe84645c0199254db8" member_type: VOTER last_known_addr { host: "127.0.96.124" port: 33751 } }
I20261120 03:06:46.399154  7978 leader_election.cc:290] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565), 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751)
W20261120 03:06:46.404327  3433 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e18f436ef9945d78f543c3471320ee7 (127.0.96.126:46565): Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:06:46.405009  3431 leader_election.cc:336] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86ed298871604dbe84645c0199254db8 (127.0.96.124:33751): Network error: Client connection negotiation failed: client connection to 127.0.96.124:33751: connect: Connection refused (error 111)
I20261120 03:06:46.405323  3431 leader_election.cc:304] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a23f052846654c9daf3fb0d90fbb6f09; no voters: 6e18f436ef9945d78f543c3471320ee7, 86ed298871604dbe84645c0199254db8
I20261120 03:06:46.405845  7978 raft_consensus.cc:2966] T 00000000000000000000000000000000 P a23f052846654c9daf3fb0d90fbb6f09 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
I20261120 03:06:46.437984  3436 maintenance_manager.cc:419] P a23f052846654c9daf3fb0d90fbb6f09: Scheduling UndoDeltaBlockGCOp(00000000000000000000000000000000): 1146 bytes on disk
I20261120 03:06:46.450380  3435 maintenance_manager.cc:643] P a23f052846654c9daf3fb0d90fbb6f09: UndoDeltaBlockGCOp(00000000000000000000000000000000) complete. Timing: real 0.012s	user 0.008s	sys 0.003s Metrics: {"dirs.queue_time_us":1707,"dirs.run_cpu_time_us":346,"dirs.run_wall_time_us":347,"thread_start_us":1991,"threads_started":2,"wal-append.queue_time_us":867}
W20261120 03:06:46.585685  2939 consensus_peers.cc:597] T 7c1a821b612b4325bc34a3a16c284b2d P c393ca4a78704704bc9b58916a81fc3f -> Peer 23741ccd042d4ca49afc3b8d1e26e22c (127.0.96.66:43733): Couldn't send request to peer 23741ccd042d4ca49afc3b8d1e26e22c. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7c1a821b612b4325bc34a3a16c284b2d. This is attempt 156: this message will repeat every 5th retry.
W20261120 03:06:47.247915  3060 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751, num_attempts: 769) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:06:47.248397  3060 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751, num_attempts: 769) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:06:47.248672  3060 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751, num_attempts: 769) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
W20261120 03:06:47.248936  3060 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751, num_attempts: 769) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
F20261120 03:06:47.259968  3077 master-stress-test.cc:393] Check failed: _s.ok() Bad status: Timed out: ReplaceTablet timed out after deadline expired: ReplaceTablet passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751, num_attempts: 769) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
*** Check failure stack trace: ***
F20261120 03:06:47.262529  3074 master-stress-test.cc:353] Check failed: _s.ok() Bad status: Timed out: DeleteTable timed out after deadline expired: DeleteTable passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751, num_attempts: 769) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
*** Check failure stack trace: ***
*** Aborted at 1795144007 (unix time) try "date -d @1795144007" if you are using GNU date ***
PC: @                0x0 (unknown)
*** SIGABRT (@0x3e800000181) received by PID 385 (TID 0x7fda3a4fd700) from PID 385; stack trace: ***
    @           0x2b0b00 __tsan::CallUserSignalHandler() at /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1915
    @           0x2b2fe3 rtl_sigaction() at /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:2005
    @     0x7fda50858980 (unknown) at ??:0
    @     0x7fda4b9f2e87 gsignal at ??:0
    @     0x7fda4b9f47f1 abort at ??:0
    @           0x2b68f6 __interceptor_abort at /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:263
    @     0x7fda4ea90d74 google::LogMessage::SendToLog() at ??:0
    @     0x7fda4ea91920 google::LogMessage::Flush() at ??:0
    @     0x7fda4ea96a5b google::LogMessageFatal::~LogMessageFatal() at ??:0
    @           0x35f5f2 kudu::MasterStressTest::ReplaceTabletThread() at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/integration-tests/master-stress-test.cc:?
    @           0x38c25e _ZNSt3__18__invokeIMN4kudu16MasterStressTestEFvvEPNS1_26MasterStressTest_Test_TestEJEvEEDTcldsdeclsr3std3__1E7forwardIT0_Efp0_Efp_spclsr3std3__1E7forwardIT1_Efp1_EEEOT_OS7_DpOS8_ at /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3846
    @           0x38c111 _ZNSt3__116__thread_executeINS_10unique_ptrINS_15__thread_structENS_14default_deleteIS2_EEEEMN4kudu16MasterStressTestEFvvEJPNS6_26MasterStressTest_Test_TestEEJLm2EEEEvRNS_5tupleIJT_T0_DpT1_EEENS_15__tuple_indicesIJXspT2_EEEE at /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/thread:281
    @           0x38b85b std::__1::__thread_proxy<>() at /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/thread:?
    @           0x2b09c0 __tsan_thread_start_func at /home/jenkins-slave/workspace/build_and_test/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:959
F20261120 03:06:47.262529  3074 master-stress-test.cc:353] Check failed: _s.ok() Bad status: Timed out: DeleteTable timed out after deadline expired: DeleteTable passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751, num_attempts: 769) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)F000F20261120 03:06:47.275779  3073 master-stress-test.cc:299] Check failed: _s.ok() Bad status: Timed out: AlterTable timed out after deadline expired: AlterTable passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751, num_attempts: 769) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
*** Check failure stack trace: ***
    @     0x7fda5084d6db start_thread at ??:0
F20261120 03:06:47.262529  3074 master-stress-test.cc:353] Check failed: _s.ok() Bad status: Timed out: DeleteTable timed out after deadline expired: DeleteTable passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751, num_attempts: 769) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)F000F20261120 03:06:47.275779  3073 master-stress-test.cc:299] Check failed: _s.ok() Bad status: Timed out: AlterTable timed out after deadline expired: AlterTable passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751, num_attempts: 769) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)F20261120 03:06:47.282639  3068 master-stress-test.cc:299] Check failed: _s.ok() Bad status: Timed out: Error creating table default.Table_ef2c937cb5564121ad7b40189665d8ba on the master: CreateTable timed out after deadline expired: CreateTable passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.0.96.126:46565,127.0.96.125:36419,127.0.96.124:33751, num_attempts: 769) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.96.126:46565: connect: Connection refused (error 111)
*** Check failure stack trace: ***
    @     0x7fda4bad561f clone at ??:0